1 //===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file This file implements the LegalizerHelper class to legalize 10 /// individual instructions and the LegalizeMachineIR wrapper pass for the 11 /// primary legalization. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h" 16 #include "llvm/CodeGen/GlobalISel/CallLowering.h" 17 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" 18 #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" 19 #include "llvm/CodeGen/MachineRegisterInfo.h" 20 #include "llvm/CodeGen/TargetFrameLowering.h" 21 #include "llvm/CodeGen/TargetInstrInfo.h" 22 #include "llvm/CodeGen/TargetLowering.h" 23 #include "llvm/CodeGen/TargetSubtargetInfo.h" 24 #include "llvm/Support/Debug.h" 25 #include "llvm/Support/MathExtras.h" 26 #include "llvm/Support/raw_ostream.h" 27 28 #define DEBUG_TYPE "legalizer" 29 30 using namespace llvm; 31 using namespace LegalizeActions; 32 33 /// Try to break down \p OrigTy into \p NarrowTy sized pieces. 34 /// 35 /// Returns the number of \p NarrowTy elements needed to reconstruct \p OrigTy, 36 /// with any leftover piece as type \p LeftoverTy 37 /// 38 /// Returns -1 in the first element of the pair if the breakdown is not 39 /// satisfiable. 40 static std::pair<int, int> 41 getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) { 42 assert(!LeftoverTy.isValid() && "this is an out argument"); 43 44 unsigned Size = OrigTy.getSizeInBits(); 45 unsigned NarrowSize = NarrowTy.getSizeInBits(); 46 unsigned NumParts = Size / NarrowSize; 47 unsigned LeftoverSize = Size - NumParts * NarrowSize; 48 assert(Size > NarrowSize); 49 50 if (LeftoverSize == 0) 51 return {NumParts, 0}; 52 53 if (NarrowTy.isVector()) { 54 unsigned EltSize = OrigTy.getScalarSizeInBits(); 55 if (LeftoverSize % EltSize != 0) 56 return {-1, -1}; 57 LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize); 58 } else { 59 LeftoverTy = LLT::scalar(LeftoverSize); 60 } 61 62 int NumLeftover = LeftoverSize / LeftoverTy.getSizeInBits(); 63 return std::make_pair(NumParts, NumLeftover); 64 } 65 66 static LLT getGCDType(LLT OrigTy, LLT TargetTy) { 67 if (OrigTy.isVector() && TargetTy.isVector()) { 68 assert(OrigTy.getElementType() == TargetTy.getElementType()); 69 int GCD = greatestCommonDivisor(OrigTy.getNumElements(), 70 TargetTy.getNumElements()); 71 return LLT::scalarOrVector(GCD, OrigTy.getElementType()); 72 } 73 74 if (OrigTy.isVector() && !TargetTy.isVector()) { 75 assert(OrigTy.getElementType() == TargetTy); 76 return TargetTy; 77 } 78 79 assert(!OrigTy.isVector() && !TargetTy.isVector() && 80 "GCD type of vector and scalar not implemented"); 81 82 int GCD = greatestCommonDivisor(OrigTy.getSizeInBits(), 83 TargetTy.getSizeInBits()); 84 return LLT::scalar(GCD); 85 } 86 87 static LLT getLCMType(LLT Ty0, LLT Ty1) { 88 if (!Ty0.isVector() && !Ty1.isVector()) { 89 unsigned Mul = Ty0.getSizeInBits() * Ty1.getSizeInBits(); 90 int GCDSize = greatestCommonDivisor(Ty0.getSizeInBits(), 91 Ty1.getSizeInBits()); 92 return LLT::scalar(Mul / GCDSize); 93 } 94 95 if (Ty0.isVector() && !Ty1.isVector()) { 96 assert(Ty0.getElementType() == Ty1 && "not yet handled"); 97 return Ty0; 98 } 99 100 if (Ty1.isVector() && !Ty0.isVector()) { 101 assert(Ty1.getElementType() == Ty0 && "not yet handled"); 102 return Ty1; 103 } 104 105 if (Ty0.isVector() && Ty1.isVector()) { 106 assert(Ty0.getElementType() == Ty1.getElementType() && "not yet handled"); 107 108 int GCDElts = greatestCommonDivisor(Ty0.getNumElements(), 109 Ty1.getNumElements()); 110 111 int Mul = Ty0.getNumElements() * Ty1.getNumElements(); 112 return LLT::vector(Mul / GCDElts, Ty0.getElementType()); 113 } 114 115 llvm_unreachable("not yet handled"); 116 } 117 118 static Type *getFloatTypeForLLT(LLVMContext &Ctx, LLT Ty) { 119 120 if (!Ty.isScalar()) 121 return nullptr; 122 123 switch (Ty.getSizeInBits()) { 124 case 16: 125 return Type::getHalfTy(Ctx); 126 case 32: 127 return Type::getFloatTy(Ctx); 128 case 64: 129 return Type::getDoubleTy(Ctx); 130 case 128: 131 return Type::getFP128Ty(Ctx); 132 default: 133 return nullptr; 134 } 135 } 136 137 LegalizerHelper::LegalizerHelper(MachineFunction &MF, 138 GISelChangeObserver &Observer, 139 MachineIRBuilder &Builder) 140 : MIRBuilder(Builder), MRI(MF.getRegInfo()), 141 LI(*MF.getSubtarget().getLegalizerInfo()), Observer(Observer) { 142 MIRBuilder.setMF(MF); 143 MIRBuilder.setChangeObserver(Observer); 144 } 145 146 LegalizerHelper::LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI, 147 GISelChangeObserver &Observer, 148 MachineIRBuilder &B) 149 : MIRBuilder(B), MRI(MF.getRegInfo()), LI(LI), Observer(Observer) { 150 MIRBuilder.setMF(MF); 151 MIRBuilder.setChangeObserver(Observer); 152 } 153 LegalizerHelper::LegalizeResult 154 LegalizerHelper::legalizeInstrStep(MachineInstr &MI) { 155 LLVM_DEBUG(dbgs() << "Legalizing: "; MI.print(dbgs())); 156 157 if (MI.getOpcode() == TargetOpcode::G_INTRINSIC || 158 MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS) 159 return LI.legalizeIntrinsic(MI, MIRBuilder, Observer) ? Legalized 160 : UnableToLegalize; 161 auto Step = LI.getAction(MI, MRI); 162 switch (Step.Action) { 163 case Legal: 164 LLVM_DEBUG(dbgs() << ".. Already legal\n"); 165 return AlreadyLegal; 166 case Libcall: 167 LLVM_DEBUG(dbgs() << ".. Convert to libcall\n"); 168 return libcall(MI); 169 case NarrowScalar: 170 LLVM_DEBUG(dbgs() << ".. Narrow scalar\n"); 171 return narrowScalar(MI, Step.TypeIdx, Step.NewType); 172 case WidenScalar: 173 LLVM_DEBUG(dbgs() << ".. Widen scalar\n"); 174 return widenScalar(MI, Step.TypeIdx, Step.NewType); 175 case Lower: 176 LLVM_DEBUG(dbgs() << ".. Lower\n"); 177 return lower(MI, Step.TypeIdx, Step.NewType); 178 case FewerElements: 179 LLVM_DEBUG(dbgs() << ".. Reduce number of elements\n"); 180 return fewerElementsVector(MI, Step.TypeIdx, Step.NewType); 181 case MoreElements: 182 LLVM_DEBUG(dbgs() << ".. Increase number of elements\n"); 183 return moreElementsVector(MI, Step.TypeIdx, Step.NewType); 184 case Custom: 185 LLVM_DEBUG(dbgs() << ".. Custom legalization\n"); 186 return LI.legalizeCustom(MI, MRI, MIRBuilder, Observer) ? Legalized 187 : UnableToLegalize; 188 default: 189 LLVM_DEBUG(dbgs() << ".. Unable to legalize\n"); 190 return UnableToLegalize; 191 } 192 } 193 194 void LegalizerHelper::extractParts(Register Reg, LLT Ty, int NumParts, 195 SmallVectorImpl<Register> &VRegs) { 196 for (int i = 0; i < NumParts; ++i) 197 VRegs.push_back(MRI.createGenericVirtualRegister(Ty)); 198 MIRBuilder.buildUnmerge(VRegs, Reg); 199 } 200 201 bool LegalizerHelper::extractParts(Register Reg, LLT RegTy, 202 LLT MainTy, LLT &LeftoverTy, 203 SmallVectorImpl<Register> &VRegs, 204 SmallVectorImpl<Register> &LeftoverRegs) { 205 assert(!LeftoverTy.isValid() && "this is an out argument"); 206 207 unsigned RegSize = RegTy.getSizeInBits(); 208 unsigned MainSize = MainTy.getSizeInBits(); 209 unsigned NumParts = RegSize / MainSize; 210 unsigned LeftoverSize = RegSize - NumParts * MainSize; 211 212 // Use an unmerge when possible. 213 if (LeftoverSize == 0) { 214 for (unsigned I = 0; I < NumParts; ++I) 215 VRegs.push_back(MRI.createGenericVirtualRegister(MainTy)); 216 MIRBuilder.buildUnmerge(VRegs, Reg); 217 return true; 218 } 219 220 if (MainTy.isVector()) { 221 unsigned EltSize = MainTy.getScalarSizeInBits(); 222 if (LeftoverSize % EltSize != 0) 223 return false; 224 LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize); 225 } else { 226 LeftoverTy = LLT::scalar(LeftoverSize); 227 } 228 229 // For irregular sizes, extract the individual parts. 230 for (unsigned I = 0; I != NumParts; ++I) { 231 Register NewReg = MRI.createGenericVirtualRegister(MainTy); 232 VRegs.push_back(NewReg); 233 MIRBuilder.buildExtract(NewReg, Reg, MainSize * I); 234 } 235 236 for (unsigned Offset = MainSize * NumParts; Offset < RegSize; 237 Offset += LeftoverSize) { 238 Register NewReg = MRI.createGenericVirtualRegister(LeftoverTy); 239 LeftoverRegs.push_back(NewReg); 240 MIRBuilder.buildExtract(NewReg, Reg, Offset); 241 } 242 243 return true; 244 } 245 246 void LegalizerHelper::insertParts(Register DstReg, 247 LLT ResultTy, LLT PartTy, 248 ArrayRef<Register> PartRegs, 249 LLT LeftoverTy, 250 ArrayRef<Register> LeftoverRegs) { 251 if (!LeftoverTy.isValid()) { 252 assert(LeftoverRegs.empty()); 253 254 if (!ResultTy.isVector()) { 255 MIRBuilder.buildMerge(DstReg, PartRegs); 256 return; 257 } 258 259 if (PartTy.isVector()) 260 MIRBuilder.buildConcatVectors(DstReg, PartRegs); 261 else 262 MIRBuilder.buildBuildVector(DstReg, PartRegs); 263 return; 264 } 265 266 unsigned PartSize = PartTy.getSizeInBits(); 267 unsigned LeftoverPartSize = LeftoverTy.getSizeInBits(); 268 269 Register CurResultReg = MRI.createGenericVirtualRegister(ResultTy); 270 MIRBuilder.buildUndef(CurResultReg); 271 272 unsigned Offset = 0; 273 for (Register PartReg : PartRegs) { 274 Register NewResultReg = MRI.createGenericVirtualRegister(ResultTy); 275 MIRBuilder.buildInsert(NewResultReg, CurResultReg, PartReg, Offset); 276 CurResultReg = NewResultReg; 277 Offset += PartSize; 278 } 279 280 for (unsigned I = 0, E = LeftoverRegs.size(); I != E; ++I) { 281 // Use the original output register for the final insert to avoid a copy. 282 Register NewResultReg = (I + 1 == E) ? 283 DstReg : MRI.createGenericVirtualRegister(ResultTy); 284 285 MIRBuilder.buildInsert(NewResultReg, CurResultReg, LeftoverRegs[I], Offset); 286 CurResultReg = NewResultReg; 287 Offset += LeftoverPartSize; 288 } 289 } 290 291 /// Return the result registers of G_UNMERGE_VALUES \p MI in \p Regs 292 static void getUnmergeResults(SmallVectorImpl<Register> &Regs, 293 const MachineInstr &MI) { 294 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES); 295 296 const int NumResults = MI.getNumOperands() - 1; 297 Regs.resize(NumResults); 298 for (int I = 0; I != NumResults; ++I) 299 Regs[I] = MI.getOperand(I).getReg(); 300 } 301 302 LLT LegalizerHelper::extractGCDType(SmallVectorImpl<Register> &Parts, LLT DstTy, 303 LLT NarrowTy, Register SrcReg) { 304 LLT SrcTy = MRI.getType(SrcReg); 305 306 LLT GCDTy = getGCDType(DstTy, getGCDType(SrcTy, NarrowTy)); 307 if (SrcTy == GCDTy) { 308 // If the source already evenly divides the result type, we don't need to do 309 // anything. 310 Parts.push_back(SrcReg); 311 } else { 312 // Need to split into common type sized pieces. 313 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg); 314 getUnmergeResults(Parts, *Unmerge); 315 } 316 317 return GCDTy; 318 } 319 320 LLT LegalizerHelper::buildLCMMergePieces(LLT DstTy, LLT NarrowTy, LLT GCDTy, 321 SmallVectorImpl<Register> &VRegs, 322 unsigned PadStrategy) { 323 LLT LCMTy = getLCMType(DstTy, NarrowTy); 324 325 int NumParts = LCMTy.getSizeInBits() / NarrowTy.getSizeInBits(); 326 int NumSubParts = NarrowTy.getSizeInBits() / GCDTy.getSizeInBits(); 327 int NumOrigSrc = VRegs.size(); 328 329 Register PadReg; 330 331 // Get a value we can use to pad the source value if the sources won't evenly 332 // cover the result type. 333 if (NumOrigSrc < NumParts * NumSubParts) { 334 if (PadStrategy == TargetOpcode::G_ZEXT) 335 PadReg = MIRBuilder.buildConstant(GCDTy, 0).getReg(0); 336 else if (PadStrategy == TargetOpcode::G_ANYEXT) 337 PadReg = MIRBuilder.buildUndef(GCDTy).getReg(0); 338 else { 339 assert(PadStrategy == TargetOpcode::G_SEXT); 340 341 // Shift the sign bit of the low register through the high register. 342 auto ShiftAmt = 343 MIRBuilder.buildConstant(LLT::scalar(64), GCDTy.getSizeInBits() - 1); 344 PadReg = MIRBuilder.buildAShr(GCDTy, VRegs.back(), ShiftAmt).getReg(0); 345 } 346 } 347 348 // Registers for the final merge to be produced. 349 SmallVector<Register, 4> Remerge(NumParts); 350 351 // Registers needed for intermediate merges, which will be merged into a 352 // source for Remerge. 353 SmallVector<Register, 4> SubMerge(NumSubParts); 354 355 // Once we've fully read off the end of the original source bits, we can reuse 356 // the same high bits for remaining padding elements. 357 Register AllPadReg; 358 359 // Build merges to the LCM type to cover the original result type. 360 for (int I = 0; I != NumParts; ++I) { 361 bool AllMergePartsArePadding = true; 362 363 // Build the requested merges to the requested type. 364 for (int J = 0; J != NumSubParts; ++J) { 365 int Idx = I * NumSubParts + J; 366 if (Idx >= NumOrigSrc) { 367 SubMerge[J] = PadReg; 368 continue; 369 } 370 371 SubMerge[J] = VRegs[Idx]; 372 373 // There are meaningful bits here we can't reuse later. 374 AllMergePartsArePadding = false; 375 } 376 377 // If we've filled up a complete piece with padding bits, we can directly 378 // emit the natural sized constant if applicable, rather than a merge of 379 // smaller constants. 380 if (AllMergePartsArePadding && !AllPadReg) { 381 if (PadStrategy == TargetOpcode::G_ANYEXT) 382 AllPadReg = MIRBuilder.buildUndef(NarrowTy).getReg(0); 383 else if (PadStrategy == TargetOpcode::G_ZEXT) 384 AllPadReg = MIRBuilder.buildConstant(NarrowTy, 0).getReg(0); 385 386 // If this is a sign extension, we can't materialize a trivial constant 387 // with the right type and have to produce a merge. 388 } 389 390 if (AllPadReg) { 391 // Avoid creating additional instructions if we're just adding additional 392 // copies of padding bits. 393 Remerge[I] = AllPadReg; 394 continue; 395 } 396 397 if (NumSubParts == 1) 398 Remerge[I] = SubMerge[0]; 399 else 400 Remerge[I] = MIRBuilder.buildMerge(NarrowTy, SubMerge).getReg(0); 401 402 // In the sign extend padding case, re-use the first all-signbit merge. 403 if (AllMergePartsArePadding && !AllPadReg) 404 AllPadReg = Remerge[I]; 405 } 406 407 VRegs = std::move(Remerge); 408 return LCMTy; 409 } 410 411 void LegalizerHelper::buildWidenedRemergeToDst(Register DstReg, LLT LCMTy, 412 ArrayRef<Register> RemergeRegs) { 413 LLT DstTy = MRI.getType(DstReg); 414 415 // Create the merge to the widened source, and extract the relevant bits into 416 // the result. 417 418 if (DstTy == LCMTy) { 419 MIRBuilder.buildMerge(DstReg, RemergeRegs); 420 return; 421 } 422 423 auto Remerge = MIRBuilder.buildMerge(LCMTy, RemergeRegs); 424 if (DstTy.isScalar() && LCMTy.isScalar()) { 425 MIRBuilder.buildTrunc(DstReg, Remerge); 426 return; 427 } 428 429 if (LCMTy.isVector()) { 430 MIRBuilder.buildExtract(DstReg, Remerge, 0); 431 return; 432 } 433 434 llvm_unreachable("unhandled case"); 435 } 436 437 static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) { 438 switch (Opcode) { 439 case TargetOpcode::G_SDIV: 440 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"); 441 switch (Size) { 442 case 32: 443 return RTLIB::SDIV_I32; 444 case 64: 445 return RTLIB::SDIV_I64; 446 case 128: 447 return RTLIB::SDIV_I128; 448 default: 449 llvm_unreachable("unexpected size"); 450 } 451 case TargetOpcode::G_UDIV: 452 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"); 453 switch (Size) { 454 case 32: 455 return RTLIB::UDIV_I32; 456 case 64: 457 return RTLIB::UDIV_I64; 458 case 128: 459 return RTLIB::UDIV_I128; 460 default: 461 llvm_unreachable("unexpected size"); 462 } 463 case TargetOpcode::G_SREM: 464 assert((Size == 32 || Size == 64) && "Unsupported size"); 465 return Size == 64 ? RTLIB::SREM_I64 : RTLIB::SREM_I32; 466 case TargetOpcode::G_UREM: 467 assert((Size == 32 || Size == 64) && "Unsupported size"); 468 return Size == 64 ? RTLIB::UREM_I64 : RTLIB::UREM_I32; 469 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 470 assert(Size == 32 && "Unsupported size"); 471 return RTLIB::CTLZ_I32; 472 case TargetOpcode::G_FADD: 473 assert((Size == 32 || Size == 64) && "Unsupported size"); 474 return Size == 64 ? RTLIB::ADD_F64 : RTLIB::ADD_F32; 475 case TargetOpcode::G_FSUB: 476 assert((Size == 32 || Size == 64) && "Unsupported size"); 477 return Size == 64 ? RTLIB::SUB_F64 : RTLIB::SUB_F32; 478 case TargetOpcode::G_FMUL: 479 assert((Size == 32 || Size == 64) && "Unsupported size"); 480 return Size == 64 ? RTLIB::MUL_F64 : RTLIB::MUL_F32; 481 case TargetOpcode::G_FDIV: 482 assert((Size == 32 || Size == 64) && "Unsupported size"); 483 return Size == 64 ? RTLIB::DIV_F64 : RTLIB::DIV_F32; 484 case TargetOpcode::G_FEXP: 485 assert((Size == 32 || Size == 64) && "Unsupported size"); 486 return Size == 64 ? RTLIB::EXP_F64 : RTLIB::EXP_F32; 487 case TargetOpcode::G_FEXP2: 488 assert((Size == 32 || Size == 64) && "Unsupported size"); 489 return Size == 64 ? RTLIB::EXP2_F64 : RTLIB::EXP2_F32; 490 case TargetOpcode::G_FREM: 491 return Size == 64 ? RTLIB::REM_F64 : RTLIB::REM_F32; 492 case TargetOpcode::G_FPOW: 493 return Size == 64 ? RTLIB::POW_F64 : RTLIB::POW_F32; 494 case TargetOpcode::G_FMA: 495 assert((Size == 32 || Size == 64) && "Unsupported size"); 496 return Size == 64 ? RTLIB::FMA_F64 : RTLIB::FMA_F32; 497 case TargetOpcode::G_FSIN: 498 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"); 499 return Size == 128 ? RTLIB::SIN_F128 500 : Size == 64 ? RTLIB::SIN_F64 : RTLIB::SIN_F32; 501 case TargetOpcode::G_FCOS: 502 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"); 503 return Size == 128 ? RTLIB::COS_F128 504 : Size == 64 ? RTLIB::COS_F64 : RTLIB::COS_F32; 505 case TargetOpcode::G_FLOG10: 506 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"); 507 return Size == 128 ? RTLIB::LOG10_F128 508 : Size == 64 ? RTLIB::LOG10_F64 : RTLIB::LOG10_F32; 509 case TargetOpcode::G_FLOG: 510 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"); 511 return Size == 128 ? RTLIB::LOG_F128 512 : Size == 64 ? RTLIB::LOG_F64 : RTLIB::LOG_F32; 513 case TargetOpcode::G_FLOG2: 514 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"); 515 return Size == 128 ? RTLIB::LOG2_F128 516 : Size == 64 ? RTLIB::LOG2_F64 : RTLIB::LOG2_F32; 517 case TargetOpcode::G_FCEIL: 518 assert((Size == 32 || Size == 64) && "Unsupported size"); 519 return Size == 64 ? RTLIB::CEIL_F64 : RTLIB::CEIL_F32; 520 case TargetOpcode::G_FFLOOR: 521 assert((Size == 32 || Size == 64) && "Unsupported size"); 522 return Size == 64 ? RTLIB::FLOOR_F64 : RTLIB::FLOOR_F32; 523 } 524 llvm_unreachable("Unknown libcall function"); 525 } 526 527 /// True if an instruction is in tail position in its caller. Intended for 528 /// legalizing libcalls as tail calls when possible. 529 static bool isLibCallInTailPosition(MachineInstr &MI) { 530 const Function &F = MI.getParent()->getParent()->getFunction(); 531 532 // Conservatively require the attributes of the call to match those of 533 // the return. Ignore NoAlias and NonNull because they don't affect the 534 // call sequence. 535 AttributeList CallerAttrs = F.getAttributes(); 536 if (AttrBuilder(CallerAttrs, AttributeList::ReturnIndex) 537 .removeAttribute(Attribute::NoAlias) 538 .removeAttribute(Attribute::NonNull) 539 .hasAttributes()) 540 return false; 541 542 // It's not safe to eliminate the sign / zero extension of the return value. 543 if (CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::ZExt) || 544 CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::SExt)) 545 return false; 546 547 // Only tail call if the following instruction is a standard return. 548 auto &TII = *MI.getMF()->getSubtarget().getInstrInfo(); 549 MachineInstr *Next = MI.getNextNode(); 550 if (!Next || TII.isTailCall(*Next) || !Next->isReturn()) 551 return false; 552 553 return true; 554 } 555 556 LegalizerHelper::LegalizeResult 557 llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall, 558 const CallLowering::ArgInfo &Result, 559 ArrayRef<CallLowering::ArgInfo> Args) { 560 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering(); 561 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering(); 562 const char *Name = TLI.getLibcallName(Libcall); 563 564 CallLowering::CallLoweringInfo Info; 565 Info.CallConv = TLI.getLibcallCallingConv(Libcall); 566 Info.Callee = MachineOperand::CreateES(Name); 567 Info.OrigRet = Result; 568 std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs)); 569 if (!CLI.lowerCall(MIRBuilder, Info)) 570 return LegalizerHelper::UnableToLegalize; 571 572 return LegalizerHelper::Legalized; 573 } 574 575 // Useful for libcalls where all operands have the same type. 576 static LegalizerHelper::LegalizeResult 577 simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size, 578 Type *OpType) { 579 auto Libcall = getRTLibDesc(MI.getOpcode(), Size); 580 581 SmallVector<CallLowering::ArgInfo, 3> Args; 582 for (unsigned i = 1; i < MI.getNumOperands(); i++) 583 Args.push_back({MI.getOperand(i).getReg(), OpType}); 584 return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), OpType}, 585 Args); 586 } 587 588 LegalizerHelper::LegalizeResult 589 llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, 590 MachineInstr &MI) { 591 assert(MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS); 592 auto &Ctx = MIRBuilder.getMF().getFunction().getContext(); 593 594 SmallVector<CallLowering::ArgInfo, 3> Args; 595 // Add all the args, except for the last which is an imm denoting 'tail'. 596 for (unsigned i = 1; i < MI.getNumOperands() - 1; i++) { 597 Register Reg = MI.getOperand(i).getReg(); 598 599 // Need derive an IR type for call lowering. 600 LLT OpLLT = MRI.getType(Reg); 601 Type *OpTy = nullptr; 602 if (OpLLT.isPointer()) 603 OpTy = Type::getInt8PtrTy(Ctx, OpLLT.getAddressSpace()); 604 else 605 OpTy = IntegerType::get(Ctx, OpLLT.getSizeInBits()); 606 Args.push_back({Reg, OpTy}); 607 } 608 609 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering(); 610 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering(); 611 Intrinsic::ID ID = MI.getOperand(0).getIntrinsicID(); 612 RTLIB::Libcall RTLibcall; 613 switch (ID) { 614 case Intrinsic::memcpy: 615 RTLibcall = RTLIB::MEMCPY; 616 break; 617 case Intrinsic::memset: 618 RTLibcall = RTLIB::MEMSET; 619 break; 620 case Intrinsic::memmove: 621 RTLibcall = RTLIB::MEMMOVE; 622 break; 623 default: 624 return LegalizerHelper::UnableToLegalize; 625 } 626 const char *Name = TLI.getLibcallName(RTLibcall); 627 628 MIRBuilder.setInstr(MI); 629 630 CallLowering::CallLoweringInfo Info; 631 Info.CallConv = TLI.getLibcallCallingConv(RTLibcall); 632 Info.Callee = MachineOperand::CreateES(Name); 633 Info.OrigRet = CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx)); 634 Info.IsTailCall = MI.getOperand(MI.getNumOperands() - 1).getImm() == 1 && 635 isLibCallInTailPosition(MI); 636 637 std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs)); 638 if (!CLI.lowerCall(MIRBuilder, Info)) 639 return LegalizerHelper::UnableToLegalize; 640 641 if (Info.LoweredTailCall) { 642 assert(Info.IsTailCall && "Lowered tail call when it wasn't a tail call?"); 643 // We must have a return following the call to get past 644 // isLibCallInTailPosition. 645 assert(MI.getNextNode() && MI.getNextNode()->isReturn() && 646 "Expected instr following MI to be a return?"); 647 648 // We lowered a tail call, so the call is now the return from the block. 649 // Delete the old return. 650 MI.getNextNode()->eraseFromParent(); 651 } 652 653 return LegalizerHelper::Legalized; 654 } 655 656 static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType, 657 Type *FromType) { 658 auto ToMVT = MVT::getVT(ToType); 659 auto FromMVT = MVT::getVT(FromType); 660 661 switch (Opcode) { 662 case TargetOpcode::G_FPEXT: 663 return RTLIB::getFPEXT(FromMVT, ToMVT); 664 case TargetOpcode::G_FPTRUNC: 665 return RTLIB::getFPROUND(FromMVT, ToMVT); 666 case TargetOpcode::G_FPTOSI: 667 return RTLIB::getFPTOSINT(FromMVT, ToMVT); 668 case TargetOpcode::G_FPTOUI: 669 return RTLIB::getFPTOUINT(FromMVT, ToMVT); 670 case TargetOpcode::G_SITOFP: 671 return RTLIB::getSINTTOFP(FromMVT, ToMVT); 672 case TargetOpcode::G_UITOFP: 673 return RTLIB::getUINTTOFP(FromMVT, ToMVT); 674 } 675 llvm_unreachable("Unsupported libcall function"); 676 } 677 678 static LegalizerHelper::LegalizeResult 679 conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType, 680 Type *FromType) { 681 RTLIB::Libcall Libcall = getConvRTLibDesc(MI.getOpcode(), ToType, FromType); 682 return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), ToType}, 683 {{MI.getOperand(1).getReg(), FromType}}); 684 } 685 686 LegalizerHelper::LegalizeResult 687 LegalizerHelper::libcall(MachineInstr &MI) { 688 LLT LLTy = MRI.getType(MI.getOperand(0).getReg()); 689 unsigned Size = LLTy.getSizeInBits(); 690 auto &Ctx = MIRBuilder.getMF().getFunction().getContext(); 691 692 MIRBuilder.setInstr(MI); 693 694 switch (MI.getOpcode()) { 695 default: 696 return UnableToLegalize; 697 case TargetOpcode::G_SDIV: 698 case TargetOpcode::G_UDIV: 699 case TargetOpcode::G_SREM: 700 case TargetOpcode::G_UREM: 701 case TargetOpcode::G_CTLZ_ZERO_UNDEF: { 702 Type *HLTy = IntegerType::get(Ctx, Size); 703 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy); 704 if (Status != Legalized) 705 return Status; 706 break; 707 } 708 case TargetOpcode::G_FADD: 709 case TargetOpcode::G_FSUB: 710 case TargetOpcode::G_FMUL: 711 case TargetOpcode::G_FDIV: 712 case TargetOpcode::G_FMA: 713 case TargetOpcode::G_FPOW: 714 case TargetOpcode::G_FREM: 715 case TargetOpcode::G_FCOS: 716 case TargetOpcode::G_FSIN: 717 case TargetOpcode::G_FLOG10: 718 case TargetOpcode::G_FLOG: 719 case TargetOpcode::G_FLOG2: 720 case TargetOpcode::G_FEXP: 721 case TargetOpcode::G_FEXP2: 722 case TargetOpcode::G_FCEIL: 723 case TargetOpcode::G_FFLOOR: { 724 Type *HLTy = getFloatTypeForLLT(Ctx, LLTy); 725 if (!HLTy || (Size != 32 && Size != 64)) { 726 LLVM_DEBUG(dbgs() << "No libcall available for size " << Size << ".\n"); 727 return UnableToLegalize; 728 } 729 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy); 730 if (Status != Legalized) 731 return Status; 732 break; 733 } 734 case TargetOpcode::G_FPEXT: 735 case TargetOpcode::G_FPTRUNC: { 736 Type *FromTy = getFloatTypeForLLT(Ctx, MRI.getType(MI.getOperand(1).getReg())); 737 Type *ToTy = getFloatTypeForLLT(Ctx, MRI.getType(MI.getOperand(0).getReg())); 738 if (!FromTy || !ToTy) 739 return UnableToLegalize; 740 LegalizeResult Status = conversionLibcall(MI, MIRBuilder, ToTy, FromTy ); 741 if (Status != Legalized) 742 return Status; 743 break; 744 } 745 case TargetOpcode::G_FPTOSI: 746 case TargetOpcode::G_FPTOUI: { 747 // FIXME: Support other types 748 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits(); 749 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 750 if ((ToSize != 32 && ToSize != 64) || (FromSize != 32 && FromSize != 64)) 751 return UnableToLegalize; 752 LegalizeResult Status = conversionLibcall( 753 MI, MIRBuilder, 754 ToSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx), 755 FromSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx)); 756 if (Status != Legalized) 757 return Status; 758 break; 759 } 760 case TargetOpcode::G_SITOFP: 761 case TargetOpcode::G_UITOFP: { 762 // FIXME: Support other types 763 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits(); 764 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 765 if ((FromSize != 32 && FromSize != 64) || (ToSize != 32 && ToSize != 64)) 766 return UnableToLegalize; 767 LegalizeResult Status = conversionLibcall( 768 MI, MIRBuilder, 769 ToSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx), 770 FromSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx)); 771 if (Status != Legalized) 772 return Status; 773 break; 774 } 775 } 776 777 MI.eraseFromParent(); 778 return Legalized; 779 } 780 781 LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI, 782 unsigned TypeIdx, 783 LLT NarrowTy) { 784 MIRBuilder.setInstr(MI); 785 786 uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 787 uint64_t NarrowSize = NarrowTy.getSizeInBits(); 788 789 switch (MI.getOpcode()) { 790 default: 791 return UnableToLegalize; 792 case TargetOpcode::G_IMPLICIT_DEF: { 793 // FIXME: add support for when SizeOp0 isn't an exact multiple of 794 // NarrowSize. 795 if (SizeOp0 % NarrowSize != 0) 796 return UnableToLegalize; 797 int NumParts = SizeOp0 / NarrowSize; 798 799 SmallVector<Register, 2> DstRegs; 800 for (int i = 0; i < NumParts; ++i) 801 DstRegs.push_back( 802 MIRBuilder.buildUndef(NarrowTy).getReg(0)); 803 804 Register DstReg = MI.getOperand(0).getReg(); 805 if(MRI.getType(DstReg).isVector()) 806 MIRBuilder.buildBuildVector(DstReg, DstRegs); 807 else 808 MIRBuilder.buildMerge(DstReg, DstRegs); 809 MI.eraseFromParent(); 810 return Legalized; 811 } 812 case TargetOpcode::G_CONSTANT: { 813 LLT Ty = MRI.getType(MI.getOperand(0).getReg()); 814 const APInt &Val = MI.getOperand(1).getCImm()->getValue(); 815 unsigned TotalSize = Ty.getSizeInBits(); 816 unsigned NarrowSize = NarrowTy.getSizeInBits(); 817 int NumParts = TotalSize / NarrowSize; 818 819 SmallVector<Register, 4> PartRegs; 820 for (int I = 0; I != NumParts; ++I) { 821 unsigned Offset = I * NarrowSize; 822 auto K = MIRBuilder.buildConstant(NarrowTy, 823 Val.lshr(Offset).trunc(NarrowSize)); 824 PartRegs.push_back(K.getReg(0)); 825 } 826 827 LLT LeftoverTy; 828 unsigned LeftoverBits = TotalSize - NumParts * NarrowSize; 829 SmallVector<Register, 1> LeftoverRegs; 830 if (LeftoverBits != 0) { 831 LeftoverTy = LLT::scalar(LeftoverBits); 832 auto K = MIRBuilder.buildConstant( 833 LeftoverTy, 834 Val.lshr(NumParts * NarrowSize).trunc(LeftoverBits)); 835 LeftoverRegs.push_back(K.getReg(0)); 836 } 837 838 insertParts(MI.getOperand(0).getReg(), 839 Ty, NarrowTy, PartRegs, LeftoverTy, LeftoverRegs); 840 841 MI.eraseFromParent(); 842 return Legalized; 843 } 844 case TargetOpcode::G_SEXT: 845 case TargetOpcode::G_ZEXT: 846 case TargetOpcode::G_ANYEXT: 847 return narrowScalarExt(MI, TypeIdx, NarrowTy); 848 case TargetOpcode::G_TRUNC: { 849 if (TypeIdx != 1) 850 return UnableToLegalize; 851 852 uint64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits(); 853 if (NarrowTy.getSizeInBits() * 2 != SizeOp1) { 854 LLVM_DEBUG(dbgs() << "Can't narrow trunc to type " << NarrowTy << "\n"); 855 return UnableToLegalize; 856 } 857 858 auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1)); 859 MIRBuilder.buildCopy(MI.getOperand(0), Unmerge.getReg(0)); 860 MI.eraseFromParent(); 861 return Legalized; 862 } 863 864 case TargetOpcode::G_ADD: { 865 // FIXME: add support for when SizeOp0 isn't an exact multiple of 866 // NarrowSize. 867 if (SizeOp0 % NarrowSize != 0) 868 return UnableToLegalize; 869 // Expand in terms of carry-setting/consuming G_ADDE instructions. 870 int NumParts = SizeOp0 / NarrowTy.getSizeInBits(); 871 872 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs; 873 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs); 874 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs); 875 876 Register CarryIn; 877 for (int i = 0; i < NumParts; ++i) { 878 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy); 879 Register CarryOut = MRI.createGenericVirtualRegister(LLT::scalar(1)); 880 881 if (i == 0) 882 MIRBuilder.buildUAddo(DstReg, CarryOut, Src1Regs[i], Src2Regs[i]); 883 else { 884 MIRBuilder.buildUAdde(DstReg, CarryOut, Src1Regs[i], 885 Src2Regs[i], CarryIn); 886 } 887 888 DstRegs.push_back(DstReg); 889 CarryIn = CarryOut; 890 } 891 Register DstReg = MI.getOperand(0).getReg(); 892 if(MRI.getType(DstReg).isVector()) 893 MIRBuilder.buildBuildVector(DstReg, DstRegs); 894 else 895 MIRBuilder.buildMerge(DstReg, DstRegs); 896 MI.eraseFromParent(); 897 return Legalized; 898 } 899 case TargetOpcode::G_SUB: { 900 // FIXME: add support for when SizeOp0 isn't an exact multiple of 901 // NarrowSize. 902 if (SizeOp0 % NarrowSize != 0) 903 return UnableToLegalize; 904 905 int NumParts = SizeOp0 / NarrowTy.getSizeInBits(); 906 907 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs; 908 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs); 909 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs); 910 911 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy); 912 Register BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1)); 913 MIRBuilder.buildInstr(TargetOpcode::G_USUBO, {DstReg, BorrowOut}, 914 {Src1Regs[0], Src2Regs[0]}); 915 DstRegs.push_back(DstReg); 916 Register BorrowIn = BorrowOut; 917 for (int i = 1; i < NumParts; ++i) { 918 DstReg = MRI.createGenericVirtualRegister(NarrowTy); 919 BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1)); 920 921 MIRBuilder.buildInstr(TargetOpcode::G_USUBE, {DstReg, BorrowOut}, 922 {Src1Regs[i], Src2Regs[i], BorrowIn}); 923 924 DstRegs.push_back(DstReg); 925 BorrowIn = BorrowOut; 926 } 927 MIRBuilder.buildMerge(MI.getOperand(0), DstRegs); 928 MI.eraseFromParent(); 929 return Legalized; 930 } 931 case TargetOpcode::G_MUL: 932 case TargetOpcode::G_UMULH: 933 return narrowScalarMul(MI, NarrowTy); 934 case TargetOpcode::G_EXTRACT: 935 return narrowScalarExtract(MI, TypeIdx, NarrowTy); 936 case TargetOpcode::G_INSERT: 937 return narrowScalarInsert(MI, TypeIdx, NarrowTy); 938 case TargetOpcode::G_LOAD: { 939 const auto &MMO = **MI.memoperands_begin(); 940 Register DstReg = MI.getOperand(0).getReg(); 941 LLT DstTy = MRI.getType(DstReg); 942 if (DstTy.isVector()) 943 return UnableToLegalize; 944 945 if (8 * MMO.getSize() != DstTy.getSizeInBits()) { 946 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy); 947 auto &MMO = **MI.memoperands_begin(); 948 MIRBuilder.buildLoad(TmpReg, MI.getOperand(1), MMO); 949 MIRBuilder.buildAnyExt(DstReg, TmpReg); 950 MI.eraseFromParent(); 951 return Legalized; 952 } 953 954 return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy); 955 } 956 case TargetOpcode::G_ZEXTLOAD: 957 case TargetOpcode::G_SEXTLOAD: { 958 bool ZExt = MI.getOpcode() == TargetOpcode::G_ZEXTLOAD; 959 Register DstReg = MI.getOperand(0).getReg(); 960 Register PtrReg = MI.getOperand(1).getReg(); 961 962 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy); 963 auto &MMO = **MI.memoperands_begin(); 964 if (MMO.getSizeInBits() == NarrowSize) { 965 MIRBuilder.buildLoad(TmpReg, PtrReg, MMO); 966 } else { 967 MIRBuilder.buildLoadInstr(MI.getOpcode(), TmpReg, PtrReg, MMO); 968 } 969 970 if (ZExt) 971 MIRBuilder.buildZExt(DstReg, TmpReg); 972 else 973 MIRBuilder.buildSExt(DstReg, TmpReg); 974 975 MI.eraseFromParent(); 976 return Legalized; 977 } 978 case TargetOpcode::G_STORE: { 979 const auto &MMO = **MI.memoperands_begin(); 980 981 Register SrcReg = MI.getOperand(0).getReg(); 982 LLT SrcTy = MRI.getType(SrcReg); 983 if (SrcTy.isVector()) 984 return UnableToLegalize; 985 986 int NumParts = SizeOp0 / NarrowSize; 987 unsigned HandledSize = NumParts * NarrowTy.getSizeInBits(); 988 unsigned LeftoverBits = SrcTy.getSizeInBits() - HandledSize; 989 if (SrcTy.isVector() && LeftoverBits != 0) 990 return UnableToLegalize; 991 992 if (8 * MMO.getSize() != SrcTy.getSizeInBits()) { 993 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy); 994 auto &MMO = **MI.memoperands_begin(); 995 MIRBuilder.buildTrunc(TmpReg, SrcReg); 996 MIRBuilder.buildStore(TmpReg, MI.getOperand(1), MMO); 997 MI.eraseFromParent(); 998 return Legalized; 999 } 1000 1001 return reduceLoadStoreWidth(MI, 0, NarrowTy); 1002 } 1003 case TargetOpcode::G_SELECT: 1004 return narrowScalarSelect(MI, TypeIdx, NarrowTy); 1005 case TargetOpcode::G_AND: 1006 case TargetOpcode::G_OR: 1007 case TargetOpcode::G_XOR: { 1008 // Legalize bitwise operation: 1009 // A = BinOp<Ty> B, C 1010 // into: 1011 // B1, ..., BN = G_UNMERGE_VALUES B 1012 // C1, ..., CN = G_UNMERGE_VALUES C 1013 // A1 = BinOp<Ty/N> B1, C2 1014 // ... 1015 // AN = BinOp<Ty/N> BN, CN 1016 // A = G_MERGE_VALUES A1, ..., AN 1017 return narrowScalarBasic(MI, TypeIdx, NarrowTy); 1018 } 1019 case TargetOpcode::G_SHL: 1020 case TargetOpcode::G_LSHR: 1021 case TargetOpcode::G_ASHR: 1022 return narrowScalarShift(MI, TypeIdx, NarrowTy); 1023 case TargetOpcode::G_CTLZ: 1024 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 1025 case TargetOpcode::G_CTTZ: 1026 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 1027 case TargetOpcode::G_CTPOP: 1028 if (TypeIdx == 1) 1029 switch (MI.getOpcode()) { 1030 case TargetOpcode::G_CTLZ: 1031 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 1032 return narrowScalarCTLZ(MI, TypeIdx, NarrowTy); 1033 case TargetOpcode::G_CTTZ: 1034 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 1035 return narrowScalarCTTZ(MI, TypeIdx, NarrowTy); 1036 case TargetOpcode::G_CTPOP: 1037 return narrowScalarCTPOP(MI, TypeIdx, NarrowTy); 1038 default: 1039 return UnableToLegalize; 1040 } 1041 1042 Observer.changingInstr(MI); 1043 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT); 1044 Observer.changedInstr(MI); 1045 return Legalized; 1046 case TargetOpcode::G_INTTOPTR: 1047 if (TypeIdx != 1) 1048 return UnableToLegalize; 1049 1050 Observer.changingInstr(MI); 1051 narrowScalarSrc(MI, NarrowTy, 1); 1052 Observer.changedInstr(MI); 1053 return Legalized; 1054 case TargetOpcode::G_PTRTOINT: 1055 if (TypeIdx != 0) 1056 return UnableToLegalize; 1057 1058 Observer.changingInstr(MI); 1059 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT); 1060 Observer.changedInstr(MI); 1061 return Legalized; 1062 case TargetOpcode::G_PHI: { 1063 unsigned NumParts = SizeOp0 / NarrowSize; 1064 SmallVector<Register, 2> DstRegs(NumParts); 1065 SmallVector<SmallVector<Register, 2>, 2> SrcRegs(MI.getNumOperands() / 2); 1066 Observer.changingInstr(MI); 1067 for (unsigned i = 1; i < MI.getNumOperands(); i += 2) { 1068 MachineBasicBlock &OpMBB = *MI.getOperand(i + 1).getMBB(); 1069 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator()); 1070 extractParts(MI.getOperand(i).getReg(), NarrowTy, NumParts, 1071 SrcRegs[i / 2]); 1072 } 1073 MachineBasicBlock &MBB = *MI.getParent(); 1074 MIRBuilder.setInsertPt(MBB, MI); 1075 for (unsigned i = 0; i < NumParts; ++i) { 1076 DstRegs[i] = MRI.createGenericVirtualRegister(NarrowTy); 1077 MachineInstrBuilder MIB = 1078 MIRBuilder.buildInstr(TargetOpcode::G_PHI).addDef(DstRegs[i]); 1079 for (unsigned j = 1; j < MI.getNumOperands(); j += 2) 1080 MIB.addUse(SrcRegs[j / 2][i]).add(MI.getOperand(j + 1)); 1081 } 1082 MIRBuilder.setInsertPt(MBB, MBB.getFirstNonPHI()); 1083 MIRBuilder.buildMerge(MI.getOperand(0), DstRegs); 1084 Observer.changedInstr(MI); 1085 MI.eraseFromParent(); 1086 return Legalized; 1087 } 1088 case TargetOpcode::G_EXTRACT_VECTOR_ELT: 1089 case TargetOpcode::G_INSERT_VECTOR_ELT: { 1090 if (TypeIdx != 2) 1091 return UnableToLegalize; 1092 1093 int OpIdx = MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3; 1094 Observer.changingInstr(MI); 1095 narrowScalarSrc(MI, NarrowTy, OpIdx); 1096 Observer.changedInstr(MI); 1097 return Legalized; 1098 } 1099 case TargetOpcode::G_ICMP: { 1100 uint64_t SrcSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits(); 1101 if (NarrowSize * 2 != SrcSize) 1102 return UnableToLegalize; 1103 1104 Observer.changingInstr(MI); 1105 Register LHSL = MRI.createGenericVirtualRegister(NarrowTy); 1106 Register LHSH = MRI.createGenericVirtualRegister(NarrowTy); 1107 MIRBuilder.buildUnmerge({LHSL, LHSH}, MI.getOperand(2)); 1108 1109 Register RHSL = MRI.createGenericVirtualRegister(NarrowTy); 1110 Register RHSH = MRI.createGenericVirtualRegister(NarrowTy); 1111 MIRBuilder.buildUnmerge({RHSL, RHSH}, MI.getOperand(3)); 1112 1113 CmpInst::Predicate Pred = 1114 static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate()); 1115 LLT ResTy = MRI.getType(MI.getOperand(0).getReg()); 1116 1117 if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE) { 1118 MachineInstrBuilder XorL = MIRBuilder.buildXor(NarrowTy, LHSL, RHSL); 1119 MachineInstrBuilder XorH = MIRBuilder.buildXor(NarrowTy, LHSH, RHSH); 1120 MachineInstrBuilder Or = MIRBuilder.buildOr(NarrowTy, XorL, XorH); 1121 MachineInstrBuilder Zero = MIRBuilder.buildConstant(NarrowTy, 0); 1122 MIRBuilder.buildICmp(Pred, MI.getOperand(0), Or, Zero); 1123 } else { 1124 MachineInstrBuilder CmpH = MIRBuilder.buildICmp(Pred, ResTy, LHSH, RHSH); 1125 MachineInstrBuilder CmpHEQ = 1126 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, ResTy, LHSH, RHSH); 1127 MachineInstrBuilder CmpLU = MIRBuilder.buildICmp( 1128 ICmpInst::getUnsignedPredicate(Pred), ResTy, LHSL, RHSL); 1129 MIRBuilder.buildSelect(MI.getOperand(0), CmpHEQ, CmpLU, CmpH); 1130 } 1131 Observer.changedInstr(MI); 1132 MI.eraseFromParent(); 1133 return Legalized; 1134 } 1135 case TargetOpcode::G_SEXT_INREG: { 1136 if (TypeIdx != 0) 1137 return UnableToLegalize; 1138 1139 int64_t SizeInBits = MI.getOperand(2).getImm(); 1140 1141 // So long as the new type has more bits than the bits we're extending we 1142 // don't need to break it apart. 1143 if (NarrowTy.getScalarSizeInBits() >= SizeInBits) { 1144 Observer.changingInstr(MI); 1145 // We don't lose any non-extension bits by truncating the src and 1146 // sign-extending the dst. 1147 MachineOperand &MO1 = MI.getOperand(1); 1148 auto TruncMIB = MIRBuilder.buildTrunc(NarrowTy, MO1); 1149 MO1.setReg(TruncMIB.getReg(0)); 1150 1151 MachineOperand &MO2 = MI.getOperand(0); 1152 Register DstExt = MRI.createGenericVirtualRegister(NarrowTy); 1153 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); 1154 MIRBuilder.buildSExt(MO2, DstExt); 1155 MO2.setReg(DstExt); 1156 Observer.changedInstr(MI); 1157 return Legalized; 1158 } 1159 1160 // Break it apart. Components below the extension point are unmodified. The 1161 // component containing the extension point becomes a narrower SEXT_INREG. 1162 // Components above it are ashr'd from the component containing the 1163 // extension point. 1164 if (SizeOp0 % NarrowSize != 0) 1165 return UnableToLegalize; 1166 int NumParts = SizeOp0 / NarrowSize; 1167 1168 // List the registers where the destination will be scattered. 1169 SmallVector<Register, 2> DstRegs; 1170 // List the registers where the source will be split. 1171 SmallVector<Register, 2> SrcRegs; 1172 1173 // Create all the temporary registers. 1174 for (int i = 0; i < NumParts; ++i) { 1175 Register SrcReg = MRI.createGenericVirtualRegister(NarrowTy); 1176 1177 SrcRegs.push_back(SrcReg); 1178 } 1179 1180 // Explode the big arguments into smaller chunks. 1181 MIRBuilder.buildUnmerge(SrcRegs, MI.getOperand(1)); 1182 1183 Register AshrCstReg = 1184 MIRBuilder.buildConstant(NarrowTy, NarrowTy.getScalarSizeInBits() - 1) 1185 .getReg(0); 1186 Register FullExtensionReg = 0; 1187 Register PartialExtensionReg = 0; 1188 1189 // Do the operation on each small part. 1190 for (int i = 0; i < NumParts; ++i) { 1191 if ((i + 1) * NarrowTy.getScalarSizeInBits() < SizeInBits) 1192 DstRegs.push_back(SrcRegs[i]); 1193 else if (i * NarrowTy.getScalarSizeInBits() > SizeInBits) { 1194 assert(PartialExtensionReg && 1195 "Expected to visit partial extension before full"); 1196 if (FullExtensionReg) { 1197 DstRegs.push_back(FullExtensionReg); 1198 continue; 1199 } 1200 DstRegs.push_back( 1201 MIRBuilder.buildAShr(NarrowTy, PartialExtensionReg, AshrCstReg) 1202 .getReg(0)); 1203 FullExtensionReg = DstRegs.back(); 1204 } else { 1205 DstRegs.push_back( 1206 MIRBuilder 1207 .buildInstr( 1208 TargetOpcode::G_SEXT_INREG, {NarrowTy}, 1209 {SrcRegs[i], SizeInBits % NarrowTy.getScalarSizeInBits()}) 1210 .getReg(0)); 1211 PartialExtensionReg = DstRegs.back(); 1212 } 1213 } 1214 1215 // Gather the destination registers into the final destination. 1216 Register DstReg = MI.getOperand(0).getReg(); 1217 MIRBuilder.buildMerge(DstReg, DstRegs); 1218 MI.eraseFromParent(); 1219 return Legalized; 1220 } 1221 case TargetOpcode::G_BSWAP: 1222 case TargetOpcode::G_BITREVERSE: { 1223 if (SizeOp0 % NarrowSize != 0) 1224 return UnableToLegalize; 1225 1226 Observer.changingInstr(MI); 1227 SmallVector<Register, 2> SrcRegs, DstRegs; 1228 unsigned NumParts = SizeOp0 / NarrowSize; 1229 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs); 1230 1231 for (unsigned i = 0; i < NumParts; ++i) { 1232 auto DstPart = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy}, 1233 {SrcRegs[NumParts - 1 - i]}); 1234 DstRegs.push_back(DstPart.getReg(0)); 1235 } 1236 1237 MIRBuilder.buildMerge(MI.getOperand(0), DstRegs); 1238 1239 Observer.changedInstr(MI); 1240 MI.eraseFromParent(); 1241 return Legalized; 1242 } 1243 } 1244 } 1245 1246 void LegalizerHelper::widenScalarSrc(MachineInstr &MI, LLT WideTy, 1247 unsigned OpIdx, unsigned ExtOpcode) { 1248 MachineOperand &MO = MI.getOperand(OpIdx); 1249 auto ExtB = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MO}); 1250 MO.setReg(ExtB.getReg(0)); 1251 } 1252 1253 void LegalizerHelper::narrowScalarSrc(MachineInstr &MI, LLT NarrowTy, 1254 unsigned OpIdx) { 1255 MachineOperand &MO = MI.getOperand(OpIdx); 1256 auto ExtB = MIRBuilder.buildTrunc(NarrowTy, MO); 1257 MO.setReg(ExtB.getReg(0)); 1258 } 1259 1260 void LegalizerHelper::widenScalarDst(MachineInstr &MI, LLT WideTy, 1261 unsigned OpIdx, unsigned TruncOpcode) { 1262 MachineOperand &MO = MI.getOperand(OpIdx); 1263 Register DstExt = MRI.createGenericVirtualRegister(WideTy); 1264 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); 1265 MIRBuilder.buildInstr(TruncOpcode, {MO}, {DstExt}); 1266 MO.setReg(DstExt); 1267 } 1268 1269 void LegalizerHelper::narrowScalarDst(MachineInstr &MI, LLT NarrowTy, 1270 unsigned OpIdx, unsigned ExtOpcode) { 1271 MachineOperand &MO = MI.getOperand(OpIdx); 1272 Register DstTrunc = MRI.createGenericVirtualRegister(NarrowTy); 1273 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); 1274 MIRBuilder.buildInstr(ExtOpcode, {MO}, {DstTrunc}); 1275 MO.setReg(DstTrunc); 1276 } 1277 1278 void LegalizerHelper::moreElementsVectorDst(MachineInstr &MI, LLT WideTy, 1279 unsigned OpIdx) { 1280 MachineOperand &MO = MI.getOperand(OpIdx); 1281 Register DstExt = MRI.createGenericVirtualRegister(WideTy); 1282 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); 1283 MIRBuilder.buildExtract(MO, DstExt, 0); 1284 MO.setReg(DstExt); 1285 } 1286 1287 void LegalizerHelper::moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy, 1288 unsigned OpIdx) { 1289 MachineOperand &MO = MI.getOperand(OpIdx); 1290 1291 LLT OldTy = MRI.getType(MO.getReg()); 1292 unsigned OldElts = OldTy.getNumElements(); 1293 unsigned NewElts = MoreTy.getNumElements(); 1294 1295 unsigned NumParts = NewElts / OldElts; 1296 1297 // Use concat_vectors if the result is a multiple of the number of elements. 1298 if (NumParts * OldElts == NewElts) { 1299 SmallVector<Register, 8> Parts; 1300 Parts.push_back(MO.getReg()); 1301 1302 Register ImpDef = MIRBuilder.buildUndef(OldTy).getReg(0); 1303 for (unsigned I = 1; I != NumParts; ++I) 1304 Parts.push_back(ImpDef); 1305 1306 auto Concat = MIRBuilder.buildConcatVectors(MoreTy, Parts); 1307 MO.setReg(Concat.getReg(0)); 1308 return; 1309 } 1310 1311 Register MoreReg = MRI.createGenericVirtualRegister(MoreTy); 1312 Register ImpDef = MIRBuilder.buildUndef(MoreTy).getReg(0); 1313 MIRBuilder.buildInsert(MoreReg, ImpDef, MO.getReg(), 0); 1314 MO.setReg(MoreReg); 1315 } 1316 1317 LegalizerHelper::LegalizeResult 1318 LegalizerHelper::widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx, 1319 LLT WideTy) { 1320 if (TypeIdx != 1) 1321 return UnableToLegalize; 1322 1323 Register DstReg = MI.getOperand(0).getReg(); 1324 LLT DstTy = MRI.getType(DstReg); 1325 if (DstTy.isVector()) 1326 return UnableToLegalize; 1327 1328 Register Src1 = MI.getOperand(1).getReg(); 1329 LLT SrcTy = MRI.getType(Src1); 1330 const int DstSize = DstTy.getSizeInBits(); 1331 const int SrcSize = SrcTy.getSizeInBits(); 1332 const int WideSize = WideTy.getSizeInBits(); 1333 const int NumMerge = (DstSize + WideSize - 1) / WideSize; 1334 1335 unsigned NumOps = MI.getNumOperands(); 1336 unsigned NumSrc = MI.getNumOperands() - 1; 1337 unsigned PartSize = DstTy.getSizeInBits() / NumSrc; 1338 1339 if (WideSize >= DstSize) { 1340 // Directly pack the bits in the target type. 1341 Register ResultReg = MIRBuilder.buildZExt(WideTy, Src1).getReg(0); 1342 1343 for (unsigned I = 2; I != NumOps; ++I) { 1344 const unsigned Offset = (I - 1) * PartSize; 1345 1346 Register SrcReg = MI.getOperand(I).getReg(); 1347 assert(MRI.getType(SrcReg) == LLT::scalar(PartSize)); 1348 1349 auto ZextInput = MIRBuilder.buildZExt(WideTy, SrcReg); 1350 1351 Register NextResult = I + 1 == NumOps && WideTy == DstTy ? DstReg : 1352 MRI.createGenericVirtualRegister(WideTy); 1353 1354 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, Offset); 1355 auto Shl = MIRBuilder.buildShl(WideTy, ZextInput, ShiftAmt); 1356 MIRBuilder.buildOr(NextResult, ResultReg, Shl); 1357 ResultReg = NextResult; 1358 } 1359 1360 if (WideSize > DstSize) 1361 MIRBuilder.buildTrunc(DstReg, ResultReg); 1362 else if (DstTy.isPointer()) 1363 MIRBuilder.buildIntToPtr(DstReg, ResultReg); 1364 1365 MI.eraseFromParent(); 1366 return Legalized; 1367 } 1368 1369 // Unmerge the original values to the GCD type, and recombine to the next 1370 // multiple greater than the original type. 1371 // 1372 // %3:_(s12) = G_MERGE_VALUES %0:_(s4), %1:_(s4), %2:_(s4) -> s6 1373 // %4:_(s2), %5:_(s2) = G_UNMERGE_VALUES %0 1374 // %6:_(s2), %7:_(s2) = G_UNMERGE_VALUES %1 1375 // %8:_(s2), %9:_(s2) = G_UNMERGE_VALUES %2 1376 // %10:_(s6) = G_MERGE_VALUES %4, %5, %6 1377 // %11:_(s6) = G_MERGE_VALUES %7, %8, %9 1378 // %12:_(s12) = G_MERGE_VALUES %10, %11 1379 // 1380 // Padding with undef if necessary: 1381 // 1382 // %2:_(s8) = G_MERGE_VALUES %0:_(s4), %1:_(s4) -> s6 1383 // %3:_(s2), %4:_(s2) = G_UNMERGE_VALUES %0 1384 // %5:_(s2), %6:_(s2) = G_UNMERGE_VALUES %1 1385 // %7:_(s2) = G_IMPLICIT_DEF 1386 // %8:_(s6) = G_MERGE_VALUES %3, %4, %5 1387 // %9:_(s6) = G_MERGE_VALUES %6, %7, %7 1388 // %10:_(s12) = G_MERGE_VALUES %8, %9 1389 1390 const int GCD = greatestCommonDivisor(SrcSize, WideSize); 1391 LLT GCDTy = LLT::scalar(GCD); 1392 1393 SmallVector<Register, 8> Parts; 1394 SmallVector<Register, 8> NewMergeRegs; 1395 SmallVector<Register, 8> Unmerges; 1396 LLT WideDstTy = LLT::scalar(NumMerge * WideSize); 1397 1398 // Decompose the original operands if they don't evenly divide. 1399 for (int I = 1, E = MI.getNumOperands(); I != E; ++I) { 1400 Register SrcReg = MI.getOperand(I).getReg(); 1401 if (GCD == SrcSize) { 1402 Unmerges.push_back(SrcReg); 1403 } else { 1404 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg); 1405 for (int J = 0, JE = Unmerge->getNumOperands() - 1; J != JE; ++J) 1406 Unmerges.push_back(Unmerge.getReg(J)); 1407 } 1408 } 1409 1410 // Pad with undef to the next size that is a multiple of the requested size. 1411 if (static_cast<int>(Unmerges.size()) != NumMerge * WideSize) { 1412 Register UndefReg = MIRBuilder.buildUndef(GCDTy).getReg(0); 1413 for (int I = Unmerges.size(); I != NumMerge * WideSize; ++I) 1414 Unmerges.push_back(UndefReg); 1415 } 1416 1417 const int PartsPerGCD = WideSize / GCD; 1418 1419 // Build merges of each piece. 1420 ArrayRef<Register> Slicer(Unmerges); 1421 for (int I = 0; I != NumMerge; ++I, Slicer = Slicer.drop_front(PartsPerGCD)) { 1422 auto Merge = MIRBuilder.buildMerge(WideTy, Slicer.take_front(PartsPerGCD)); 1423 NewMergeRegs.push_back(Merge.getReg(0)); 1424 } 1425 1426 // A truncate may be necessary if the requested type doesn't evenly divide the 1427 // original result type. 1428 if (DstTy.getSizeInBits() == WideDstTy.getSizeInBits()) { 1429 MIRBuilder.buildMerge(DstReg, NewMergeRegs); 1430 } else { 1431 auto FinalMerge = MIRBuilder.buildMerge(WideDstTy, NewMergeRegs); 1432 MIRBuilder.buildTrunc(DstReg, FinalMerge.getReg(0)); 1433 } 1434 1435 MI.eraseFromParent(); 1436 return Legalized; 1437 } 1438 1439 LegalizerHelper::LegalizeResult 1440 LegalizerHelper::widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx, 1441 LLT WideTy) { 1442 if (TypeIdx != 0) 1443 return UnableToLegalize; 1444 1445 int NumDst = MI.getNumOperands() - 1; 1446 Register SrcReg = MI.getOperand(NumDst).getReg(); 1447 LLT SrcTy = MRI.getType(SrcReg); 1448 if (SrcTy.isVector()) 1449 return UnableToLegalize; 1450 1451 Register Dst0Reg = MI.getOperand(0).getReg(); 1452 LLT DstTy = MRI.getType(Dst0Reg); 1453 if (!DstTy.isScalar()) 1454 return UnableToLegalize; 1455 1456 if (WideTy.getSizeInBits() == SrcTy.getSizeInBits()) { 1457 if (SrcTy.isPointer()) { 1458 const DataLayout &DL = MIRBuilder.getDataLayout(); 1459 if (DL.isNonIntegralAddressSpace(SrcTy.getAddressSpace())) { 1460 LLVM_DEBUG(dbgs() << "Not casting non-integral address space integer\n"); 1461 return UnableToLegalize; 1462 } 1463 1464 SrcTy = LLT::scalar(SrcTy.getSizeInBits()); 1465 SrcReg = MIRBuilder.buildPtrToInt(SrcTy, SrcReg).getReg(0); 1466 } 1467 1468 // Theres no unmerge type to target. Directly extract the bits from the 1469 // source type 1470 unsigned DstSize = DstTy.getSizeInBits(); 1471 1472 MIRBuilder.buildTrunc(Dst0Reg, SrcReg); 1473 for (int I = 1; I != NumDst; ++I) { 1474 auto ShiftAmt = MIRBuilder.buildConstant(SrcTy, DstSize * I); 1475 auto Shr = MIRBuilder.buildLShr(SrcTy, SrcReg, ShiftAmt); 1476 MIRBuilder.buildTrunc(MI.getOperand(I), Shr); 1477 } 1478 1479 MI.eraseFromParent(); 1480 return Legalized; 1481 } 1482 1483 // TODO 1484 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) 1485 return UnableToLegalize; 1486 1487 // Extend the source to a wider type. 1488 LLT LCMTy = getLCMType(SrcTy, WideTy); 1489 1490 Register WideSrc = SrcReg; 1491 if (LCMTy.getSizeInBits() != SrcTy.getSizeInBits()) { 1492 // TODO: If this is an integral address space, cast to integer and anyext. 1493 if (SrcTy.isPointer()) { 1494 LLVM_DEBUG(dbgs() << "Widening pointer source types not implemented\n"); 1495 return UnableToLegalize; 1496 } 1497 1498 WideSrc = MIRBuilder.buildAnyExt(LCMTy, WideSrc).getReg(0); 1499 } 1500 1501 auto Unmerge = MIRBuilder.buildUnmerge(WideTy, WideSrc); 1502 1503 // Create a sequence of unmerges to the original results. since we may have 1504 // widened the source, we will need to pad the results with dead defs to cover 1505 // the source register. 1506 // e.g. widen s16 to s32: 1507 // %1:_(s16), %2:_(s16), %3:_(s16) = G_UNMERGE_VALUES %0:_(s48) 1508 // 1509 // => 1510 // %4:_(s64) = G_ANYEXT %0:_(s48) 1511 // %5:_(s32), %6:_(s32) = G_UNMERGE_VALUES %4 ; Requested unmerge 1512 // %1:_(s16), %2:_(s16) = G_UNMERGE_VALUES %5 ; unpack to original regs 1513 // %3:_(s16), dead %7 = G_UNMERGE_VALUES %6 ; original reg + extra dead def 1514 1515 const int NumUnmerge = Unmerge->getNumOperands() - 1; 1516 const int PartsPerUnmerge = WideTy.getSizeInBits() / DstTy.getSizeInBits(); 1517 1518 for (int I = 0; I != NumUnmerge; ++I) { 1519 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES); 1520 1521 for (int J = 0; J != PartsPerUnmerge; ++J) { 1522 int Idx = I * PartsPerUnmerge + J; 1523 if (Idx < NumDst) 1524 MIB.addDef(MI.getOperand(Idx).getReg()); 1525 else { 1526 // Create dead def for excess components. 1527 MIB.addDef(MRI.createGenericVirtualRegister(DstTy)); 1528 } 1529 } 1530 1531 MIB.addUse(Unmerge.getReg(I)); 1532 } 1533 1534 MI.eraseFromParent(); 1535 return Legalized; 1536 } 1537 1538 LegalizerHelper::LegalizeResult 1539 LegalizerHelper::widenScalarExtract(MachineInstr &MI, unsigned TypeIdx, 1540 LLT WideTy) { 1541 Register DstReg = MI.getOperand(0).getReg(); 1542 Register SrcReg = MI.getOperand(1).getReg(); 1543 LLT SrcTy = MRI.getType(SrcReg); 1544 1545 LLT DstTy = MRI.getType(DstReg); 1546 unsigned Offset = MI.getOperand(2).getImm(); 1547 1548 if (TypeIdx == 0) { 1549 if (SrcTy.isVector() || DstTy.isVector()) 1550 return UnableToLegalize; 1551 1552 SrcOp Src(SrcReg); 1553 if (SrcTy.isPointer()) { 1554 // Extracts from pointers can be handled only if they are really just 1555 // simple integers. 1556 const DataLayout &DL = MIRBuilder.getDataLayout(); 1557 if (DL.isNonIntegralAddressSpace(SrcTy.getAddressSpace())) 1558 return UnableToLegalize; 1559 1560 LLT SrcAsIntTy = LLT::scalar(SrcTy.getSizeInBits()); 1561 Src = MIRBuilder.buildPtrToInt(SrcAsIntTy, Src); 1562 SrcTy = SrcAsIntTy; 1563 } 1564 1565 if (DstTy.isPointer()) 1566 return UnableToLegalize; 1567 1568 if (Offset == 0) { 1569 // Avoid a shift in the degenerate case. 1570 MIRBuilder.buildTrunc(DstReg, 1571 MIRBuilder.buildAnyExtOrTrunc(WideTy, Src)); 1572 MI.eraseFromParent(); 1573 return Legalized; 1574 } 1575 1576 // Do a shift in the source type. 1577 LLT ShiftTy = SrcTy; 1578 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) { 1579 Src = MIRBuilder.buildAnyExt(WideTy, Src); 1580 ShiftTy = WideTy; 1581 } else if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) 1582 return UnableToLegalize; 1583 1584 auto LShr = MIRBuilder.buildLShr( 1585 ShiftTy, Src, MIRBuilder.buildConstant(ShiftTy, Offset)); 1586 MIRBuilder.buildTrunc(DstReg, LShr); 1587 MI.eraseFromParent(); 1588 return Legalized; 1589 } 1590 1591 if (SrcTy.isScalar()) { 1592 Observer.changingInstr(MI); 1593 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); 1594 Observer.changedInstr(MI); 1595 return Legalized; 1596 } 1597 1598 if (!SrcTy.isVector()) 1599 return UnableToLegalize; 1600 1601 if (DstTy != SrcTy.getElementType()) 1602 return UnableToLegalize; 1603 1604 if (Offset % SrcTy.getScalarSizeInBits() != 0) 1605 return UnableToLegalize; 1606 1607 Observer.changingInstr(MI); 1608 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); 1609 1610 MI.getOperand(2).setImm((WideTy.getSizeInBits() / SrcTy.getSizeInBits()) * 1611 Offset); 1612 widenScalarDst(MI, WideTy.getScalarType(), 0); 1613 Observer.changedInstr(MI); 1614 return Legalized; 1615 } 1616 1617 LegalizerHelper::LegalizeResult 1618 LegalizerHelper::widenScalarInsert(MachineInstr &MI, unsigned TypeIdx, 1619 LLT WideTy) { 1620 if (TypeIdx != 0) 1621 return UnableToLegalize; 1622 Observer.changingInstr(MI); 1623 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); 1624 widenScalarDst(MI, WideTy); 1625 Observer.changedInstr(MI); 1626 return Legalized; 1627 } 1628 1629 LegalizerHelper::LegalizeResult 1630 LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { 1631 MIRBuilder.setInstr(MI); 1632 1633 switch (MI.getOpcode()) { 1634 default: 1635 return UnableToLegalize; 1636 case TargetOpcode::G_EXTRACT: 1637 return widenScalarExtract(MI, TypeIdx, WideTy); 1638 case TargetOpcode::G_INSERT: 1639 return widenScalarInsert(MI, TypeIdx, WideTy); 1640 case TargetOpcode::G_MERGE_VALUES: 1641 return widenScalarMergeValues(MI, TypeIdx, WideTy); 1642 case TargetOpcode::G_UNMERGE_VALUES: 1643 return widenScalarUnmergeValues(MI, TypeIdx, WideTy); 1644 case TargetOpcode::G_UADDO: 1645 case TargetOpcode::G_USUBO: { 1646 if (TypeIdx == 1) 1647 return UnableToLegalize; // TODO 1648 auto LHSZext = MIRBuilder.buildZExt(WideTy, MI.getOperand(2)); 1649 auto RHSZext = MIRBuilder.buildZExt(WideTy, MI.getOperand(3)); 1650 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_UADDO 1651 ? TargetOpcode::G_ADD 1652 : TargetOpcode::G_SUB; 1653 // Do the arithmetic in the larger type. 1654 auto NewOp = MIRBuilder.buildInstr(Opcode, {WideTy}, {LHSZext, RHSZext}); 1655 LLT OrigTy = MRI.getType(MI.getOperand(0).getReg()); 1656 APInt Mask = 1657 APInt::getLowBitsSet(WideTy.getSizeInBits(), OrigTy.getSizeInBits()); 1658 auto AndOp = MIRBuilder.buildAnd( 1659 WideTy, NewOp, MIRBuilder.buildConstant(WideTy, Mask)); 1660 // There is no overflow if the AndOp is the same as NewOp. 1661 MIRBuilder.buildICmp(CmpInst::ICMP_NE, MI.getOperand(1), NewOp, AndOp); 1662 // Now trunc the NewOp to the original result. 1663 MIRBuilder.buildTrunc(MI.getOperand(0), NewOp); 1664 MI.eraseFromParent(); 1665 return Legalized; 1666 } 1667 case TargetOpcode::G_CTTZ: 1668 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 1669 case TargetOpcode::G_CTLZ: 1670 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 1671 case TargetOpcode::G_CTPOP: { 1672 if (TypeIdx == 0) { 1673 Observer.changingInstr(MI); 1674 widenScalarDst(MI, WideTy, 0); 1675 Observer.changedInstr(MI); 1676 return Legalized; 1677 } 1678 1679 Register SrcReg = MI.getOperand(1).getReg(); 1680 1681 // First ZEXT the input. 1682 auto MIBSrc = MIRBuilder.buildZExt(WideTy, SrcReg); 1683 LLT CurTy = MRI.getType(SrcReg); 1684 if (MI.getOpcode() == TargetOpcode::G_CTTZ) { 1685 // The count is the same in the larger type except if the original 1686 // value was zero. This can be handled by setting the bit just off 1687 // the top of the original type. 1688 auto TopBit = 1689 APInt::getOneBitSet(WideTy.getSizeInBits(), CurTy.getSizeInBits()); 1690 MIBSrc = MIRBuilder.buildOr( 1691 WideTy, MIBSrc, MIRBuilder.buildConstant(WideTy, TopBit)); 1692 } 1693 1694 // Perform the operation at the larger size. 1695 auto MIBNewOp = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy}, {MIBSrc}); 1696 // This is already the correct result for CTPOP and CTTZs 1697 if (MI.getOpcode() == TargetOpcode::G_CTLZ || 1698 MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF) { 1699 // The correct result is NewOp - (Difference in widety and current ty). 1700 unsigned SizeDiff = WideTy.getSizeInBits() - CurTy.getSizeInBits(); 1701 MIBNewOp = MIRBuilder.buildSub( 1702 WideTy, MIBNewOp, MIRBuilder.buildConstant(WideTy, SizeDiff)); 1703 } 1704 1705 MIRBuilder.buildZExtOrTrunc(MI.getOperand(0), MIBNewOp); 1706 MI.eraseFromParent(); 1707 return Legalized; 1708 } 1709 case TargetOpcode::G_BSWAP: { 1710 Observer.changingInstr(MI); 1711 Register DstReg = MI.getOperand(0).getReg(); 1712 1713 Register ShrReg = MRI.createGenericVirtualRegister(WideTy); 1714 Register DstExt = MRI.createGenericVirtualRegister(WideTy); 1715 Register ShiftAmtReg = MRI.createGenericVirtualRegister(WideTy); 1716 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); 1717 1718 MI.getOperand(0).setReg(DstExt); 1719 1720 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); 1721 1722 LLT Ty = MRI.getType(DstReg); 1723 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits(); 1724 MIRBuilder.buildConstant(ShiftAmtReg, DiffBits); 1725 MIRBuilder.buildLShr(ShrReg, DstExt, ShiftAmtReg); 1726 1727 MIRBuilder.buildTrunc(DstReg, ShrReg); 1728 Observer.changedInstr(MI); 1729 return Legalized; 1730 } 1731 case TargetOpcode::G_BITREVERSE: { 1732 Observer.changingInstr(MI); 1733 1734 Register DstReg = MI.getOperand(0).getReg(); 1735 LLT Ty = MRI.getType(DstReg); 1736 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits(); 1737 1738 Register DstExt = MRI.createGenericVirtualRegister(WideTy); 1739 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); 1740 MI.getOperand(0).setReg(DstExt); 1741 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); 1742 1743 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, DiffBits); 1744 auto Shift = MIRBuilder.buildLShr(WideTy, DstExt, ShiftAmt); 1745 MIRBuilder.buildTrunc(DstReg, Shift); 1746 Observer.changedInstr(MI); 1747 return Legalized; 1748 } 1749 case TargetOpcode::G_ADD: 1750 case TargetOpcode::G_AND: 1751 case TargetOpcode::G_MUL: 1752 case TargetOpcode::G_OR: 1753 case TargetOpcode::G_XOR: 1754 case TargetOpcode::G_SUB: 1755 // Perform operation at larger width (any extension is fines here, high bits 1756 // don't affect the result) and then truncate the result back to the 1757 // original type. 1758 Observer.changingInstr(MI); 1759 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); 1760 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT); 1761 widenScalarDst(MI, WideTy); 1762 Observer.changedInstr(MI); 1763 return Legalized; 1764 1765 case TargetOpcode::G_SHL: 1766 Observer.changingInstr(MI); 1767 1768 if (TypeIdx == 0) { 1769 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); 1770 widenScalarDst(MI, WideTy); 1771 } else { 1772 assert(TypeIdx == 1); 1773 // The "number of bits to shift" operand must preserve its value as an 1774 // unsigned integer: 1775 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); 1776 } 1777 1778 Observer.changedInstr(MI); 1779 return Legalized; 1780 1781 case TargetOpcode::G_SDIV: 1782 case TargetOpcode::G_SREM: 1783 case TargetOpcode::G_SMIN: 1784 case TargetOpcode::G_SMAX: 1785 Observer.changingInstr(MI); 1786 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT); 1787 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT); 1788 widenScalarDst(MI, WideTy); 1789 Observer.changedInstr(MI); 1790 return Legalized; 1791 1792 case TargetOpcode::G_ASHR: 1793 case TargetOpcode::G_LSHR: 1794 Observer.changingInstr(MI); 1795 1796 if (TypeIdx == 0) { 1797 unsigned CvtOp = MI.getOpcode() == TargetOpcode::G_ASHR ? 1798 TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT; 1799 1800 widenScalarSrc(MI, WideTy, 1, CvtOp); 1801 widenScalarDst(MI, WideTy); 1802 } else { 1803 assert(TypeIdx == 1); 1804 // The "number of bits to shift" operand must preserve its value as an 1805 // unsigned integer: 1806 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); 1807 } 1808 1809 Observer.changedInstr(MI); 1810 return Legalized; 1811 case TargetOpcode::G_UDIV: 1812 case TargetOpcode::G_UREM: 1813 case TargetOpcode::G_UMIN: 1814 case TargetOpcode::G_UMAX: 1815 Observer.changingInstr(MI); 1816 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT); 1817 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); 1818 widenScalarDst(MI, WideTy); 1819 Observer.changedInstr(MI); 1820 return Legalized; 1821 1822 case TargetOpcode::G_SELECT: 1823 Observer.changingInstr(MI); 1824 if (TypeIdx == 0) { 1825 // Perform operation at larger width (any extension is fine here, high 1826 // bits don't affect the result) and then truncate the result back to the 1827 // original type. 1828 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT); 1829 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT); 1830 widenScalarDst(MI, WideTy); 1831 } else { 1832 bool IsVec = MRI.getType(MI.getOperand(1).getReg()).isVector(); 1833 // Explicit extension is required here since high bits affect the result. 1834 widenScalarSrc(MI, WideTy, 1, MIRBuilder.getBoolExtOp(IsVec, false)); 1835 } 1836 Observer.changedInstr(MI); 1837 return Legalized; 1838 1839 case TargetOpcode::G_FPTOSI: 1840 case TargetOpcode::G_FPTOUI: 1841 Observer.changingInstr(MI); 1842 1843 if (TypeIdx == 0) 1844 widenScalarDst(MI, WideTy); 1845 else 1846 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT); 1847 1848 Observer.changedInstr(MI); 1849 return Legalized; 1850 case TargetOpcode::G_SITOFP: 1851 if (TypeIdx != 1) 1852 return UnableToLegalize; 1853 Observer.changingInstr(MI); 1854 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT); 1855 Observer.changedInstr(MI); 1856 return Legalized; 1857 1858 case TargetOpcode::G_UITOFP: 1859 if (TypeIdx != 1) 1860 return UnableToLegalize; 1861 Observer.changingInstr(MI); 1862 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT); 1863 Observer.changedInstr(MI); 1864 return Legalized; 1865 1866 case TargetOpcode::G_LOAD: 1867 case TargetOpcode::G_SEXTLOAD: 1868 case TargetOpcode::G_ZEXTLOAD: 1869 Observer.changingInstr(MI); 1870 widenScalarDst(MI, WideTy); 1871 Observer.changedInstr(MI); 1872 return Legalized; 1873 1874 case TargetOpcode::G_STORE: { 1875 if (TypeIdx != 0) 1876 return UnableToLegalize; 1877 1878 LLT Ty = MRI.getType(MI.getOperand(0).getReg()); 1879 if (!isPowerOf2_32(Ty.getSizeInBits())) 1880 return UnableToLegalize; 1881 1882 Observer.changingInstr(MI); 1883 1884 unsigned ExtType = Ty.getScalarSizeInBits() == 1 ? 1885 TargetOpcode::G_ZEXT : TargetOpcode::G_ANYEXT; 1886 widenScalarSrc(MI, WideTy, 0, ExtType); 1887 1888 Observer.changedInstr(MI); 1889 return Legalized; 1890 } 1891 case TargetOpcode::G_CONSTANT: { 1892 MachineOperand &SrcMO = MI.getOperand(1); 1893 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext(); 1894 unsigned ExtOpc = LI.getExtOpcodeForWideningConstant( 1895 MRI.getType(MI.getOperand(0).getReg())); 1896 assert((ExtOpc == TargetOpcode::G_ZEXT || ExtOpc == TargetOpcode::G_SEXT || 1897 ExtOpc == TargetOpcode::G_ANYEXT) && 1898 "Illegal Extend"); 1899 const APInt &SrcVal = SrcMO.getCImm()->getValue(); 1900 const APInt &Val = (ExtOpc == TargetOpcode::G_SEXT) 1901 ? SrcVal.sext(WideTy.getSizeInBits()) 1902 : SrcVal.zext(WideTy.getSizeInBits()); 1903 Observer.changingInstr(MI); 1904 SrcMO.setCImm(ConstantInt::get(Ctx, Val)); 1905 1906 widenScalarDst(MI, WideTy); 1907 Observer.changedInstr(MI); 1908 return Legalized; 1909 } 1910 case TargetOpcode::G_FCONSTANT: { 1911 MachineOperand &SrcMO = MI.getOperand(1); 1912 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext(); 1913 APFloat Val = SrcMO.getFPImm()->getValueAPF(); 1914 bool LosesInfo; 1915 switch (WideTy.getSizeInBits()) { 1916 case 32: 1917 Val.convert(APFloat::IEEEsingle(), APFloat::rmNearestTiesToEven, 1918 &LosesInfo); 1919 break; 1920 case 64: 1921 Val.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven, 1922 &LosesInfo); 1923 break; 1924 default: 1925 return UnableToLegalize; 1926 } 1927 1928 assert(!LosesInfo && "extend should always be lossless"); 1929 1930 Observer.changingInstr(MI); 1931 SrcMO.setFPImm(ConstantFP::get(Ctx, Val)); 1932 1933 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC); 1934 Observer.changedInstr(MI); 1935 return Legalized; 1936 } 1937 case TargetOpcode::G_IMPLICIT_DEF: { 1938 Observer.changingInstr(MI); 1939 widenScalarDst(MI, WideTy); 1940 Observer.changedInstr(MI); 1941 return Legalized; 1942 } 1943 case TargetOpcode::G_BRCOND: 1944 Observer.changingInstr(MI); 1945 widenScalarSrc(MI, WideTy, 0, MIRBuilder.getBoolExtOp(false, false)); 1946 Observer.changedInstr(MI); 1947 return Legalized; 1948 1949 case TargetOpcode::G_FCMP: 1950 Observer.changingInstr(MI); 1951 if (TypeIdx == 0) 1952 widenScalarDst(MI, WideTy); 1953 else { 1954 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT); 1955 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_FPEXT); 1956 } 1957 Observer.changedInstr(MI); 1958 return Legalized; 1959 1960 case TargetOpcode::G_ICMP: 1961 Observer.changingInstr(MI); 1962 if (TypeIdx == 0) 1963 widenScalarDst(MI, WideTy); 1964 else { 1965 unsigned ExtOpcode = CmpInst::isSigned(static_cast<CmpInst::Predicate>( 1966 MI.getOperand(1).getPredicate())) 1967 ? TargetOpcode::G_SEXT 1968 : TargetOpcode::G_ZEXT; 1969 widenScalarSrc(MI, WideTy, 2, ExtOpcode); 1970 widenScalarSrc(MI, WideTy, 3, ExtOpcode); 1971 } 1972 Observer.changedInstr(MI); 1973 return Legalized; 1974 1975 case TargetOpcode::G_PTR_ADD: 1976 assert(TypeIdx == 1 && "unable to legalize pointer of G_PTR_ADD"); 1977 Observer.changingInstr(MI); 1978 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT); 1979 Observer.changedInstr(MI); 1980 return Legalized; 1981 1982 case TargetOpcode::G_PHI: { 1983 assert(TypeIdx == 0 && "Expecting only Idx 0"); 1984 1985 Observer.changingInstr(MI); 1986 for (unsigned I = 1; I < MI.getNumOperands(); I += 2) { 1987 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB(); 1988 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator()); 1989 widenScalarSrc(MI, WideTy, I, TargetOpcode::G_ANYEXT); 1990 } 1991 1992 MachineBasicBlock &MBB = *MI.getParent(); 1993 MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI()); 1994 widenScalarDst(MI, WideTy); 1995 Observer.changedInstr(MI); 1996 return Legalized; 1997 } 1998 case TargetOpcode::G_EXTRACT_VECTOR_ELT: { 1999 if (TypeIdx == 0) { 2000 Register VecReg = MI.getOperand(1).getReg(); 2001 LLT VecTy = MRI.getType(VecReg); 2002 Observer.changingInstr(MI); 2003 2004 widenScalarSrc(MI, LLT::vector(VecTy.getNumElements(), 2005 WideTy.getSizeInBits()), 2006 1, TargetOpcode::G_SEXT); 2007 2008 widenScalarDst(MI, WideTy, 0); 2009 Observer.changedInstr(MI); 2010 return Legalized; 2011 } 2012 2013 if (TypeIdx != 2) 2014 return UnableToLegalize; 2015 Observer.changingInstr(MI); 2016 // TODO: Probably should be zext 2017 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT); 2018 Observer.changedInstr(MI); 2019 return Legalized; 2020 } 2021 case TargetOpcode::G_INSERT_VECTOR_ELT: { 2022 if (TypeIdx == 1) { 2023 Observer.changingInstr(MI); 2024 2025 Register VecReg = MI.getOperand(1).getReg(); 2026 LLT VecTy = MRI.getType(VecReg); 2027 LLT WideVecTy = LLT::vector(VecTy.getNumElements(), WideTy); 2028 2029 widenScalarSrc(MI, WideVecTy, 1, TargetOpcode::G_ANYEXT); 2030 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT); 2031 widenScalarDst(MI, WideVecTy, 0); 2032 Observer.changedInstr(MI); 2033 return Legalized; 2034 } 2035 2036 if (TypeIdx == 2) { 2037 Observer.changingInstr(MI); 2038 // TODO: Probably should be zext 2039 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_SEXT); 2040 Observer.changedInstr(MI); 2041 } 2042 2043 return Legalized; 2044 } 2045 case TargetOpcode::G_FADD: 2046 case TargetOpcode::G_FMUL: 2047 case TargetOpcode::G_FSUB: 2048 case TargetOpcode::G_FMA: 2049 case TargetOpcode::G_FMAD: 2050 case TargetOpcode::G_FNEG: 2051 case TargetOpcode::G_FABS: 2052 case TargetOpcode::G_FCANONICALIZE: 2053 case TargetOpcode::G_FMINNUM: 2054 case TargetOpcode::G_FMAXNUM: 2055 case TargetOpcode::G_FMINNUM_IEEE: 2056 case TargetOpcode::G_FMAXNUM_IEEE: 2057 case TargetOpcode::G_FMINIMUM: 2058 case TargetOpcode::G_FMAXIMUM: 2059 case TargetOpcode::G_FDIV: 2060 case TargetOpcode::G_FREM: 2061 case TargetOpcode::G_FCEIL: 2062 case TargetOpcode::G_FFLOOR: 2063 case TargetOpcode::G_FCOS: 2064 case TargetOpcode::G_FSIN: 2065 case TargetOpcode::G_FLOG10: 2066 case TargetOpcode::G_FLOG: 2067 case TargetOpcode::G_FLOG2: 2068 case TargetOpcode::G_FRINT: 2069 case TargetOpcode::G_FNEARBYINT: 2070 case TargetOpcode::G_FSQRT: 2071 case TargetOpcode::G_FEXP: 2072 case TargetOpcode::G_FEXP2: 2073 case TargetOpcode::G_FPOW: 2074 case TargetOpcode::G_INTRINSIC_TRUNC: 2075 case TargetOpcode::G_INTRINSIC_ROUND: 2076 assert(TypeIdx == 0); 2077 Observer.changingInstr(MI); 2078 2079 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) 2080 widenScalarSrc(MI, WideTy, I, TargetOpcode::G_FPEXT); 2081 2082 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC); 2083 Observer.changedInstr(MI); 2084 return Legalized; 2085 case TargetOpcode::G_INTTOPTR: 2086 if (TypeIdx != 1) 2087 return UnableToLegalize; 2088 2089 Observer.changingInstr(MI); 2090 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT); 2091 Observer.changedInstr(MI); 2092 return Legalized; 2093 case TargetOpcode::G_PTRTOINT: 2094 if (TypeIdx != 0) 2095 return UnableToLegalize; 2096 2097 Observer.changingInstr(MI); 2098 widenScalarDst(MI, WideTy, 0); 2099 Observer.changedInstr(MI); 2100 return Legalized; 2101 case TargetOpcode::G_BUILD_VECTOR: { 2102 Observer.changingInstr(MI); 2103 2104 const LLT WideEltTy = TypeIdx == 1 ? WideTy : WideTy.getElementType(); 2105 for (int I = 1, E = MI.getNumOperands(); I != E; ++I) 2106 widenScalarSrc(MI, WideEltTy, I, TargetOpcode::G_ANYEXT); 2107 2108 // Avoid changing the result vector type if the source element type was 2109 // requested. 2110 if (TypeIdx == 1) { 2111 auto &TII = *MI.getMF()->getSubtarget().getInstrInfo(); 2112 MI.setDesc(TII.get(TargetOpcode::G_BUILD_VECTOR_TRUNC)); 2113 } else { 2114 widenScalarDst(MI, WideTy, 0); 2115 } 2116 2117 Observer.changedInstr(MI); 2118 return Legalized; 2119 } 2120 case TargetOpcode::G_SEXT_INREG: 2121 if (TypeIdx != 0) 2122 return UnableToLegalize; 2123 2124 Observer.changingInstr(MI); 2125 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); 2126 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_TRUNC); 2127 Observer.changedInstr(MI); 2128 return Legalized; 2129 } 2130 } 2131 2132 static void getUnmergePieces(SmallVectorImpl<Register> &Pieces, 2133 MachineIRBuilder &B, Register Src, LLT Ty) { 2134 auto Unmerge = B.buildUnmerge(Ty, Src); 2135 for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I) 2136 Pieces.push_back(Unmerge.getReg(I)); 2137 } 2138 2139 LegalizerHelper::LegalizeResult 2140 LegalizerHelper::lowerBitcast(MachineInstr &MI) { 2141 Register Dst = MI.getOperand(0).getReg(); 2142 Register Src = MI.getOperand(1).getReg(); 2143 LLT DstTy = MRI.getType(Dst); 2144 LLT SrcTy = MRI.getType(Src); 2145 2146 if (SrcTy.isVector() && !DstTy.isVector()) { 2147 SmallVector<Register, 8> SrcRegs; 2148 getUnmergePieces(SrcRegs, MIRBuilder, Src, SrcTy.getElementType()); 2149 MIRBuilder.buildMerge(Dst, SrcRegs); 2150 MI.eraseFromParent(); 2151 return Legalized; 2152 } 2153 2154 if (DstTy.isVector() && !SrcTy.isVector()) { 2155 SmallVector<Register, 8> SrcRegs; 2156 getUnmergePieces(SrcRegs, MIRBuilder, Src, DstTy.getElementType()); 2157 MIRBuilder.buildMerge(Dst, SrcRegs); 2158 MI.eraseFromParent(); 2159 return Legalized; 2160 } 2161 2162 return UnableToLegalize; 2163 } 2164 2165 LegalizerHelper::LegalizeResult 2166 LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { 2167 using namespace TargetOpcode; 2168 MIRBuilder.setInstr(MI); 2169 2170 switch(MI.getOpcode()) { 2171 default: 2172 return UnableToLegalize; 2173 case TargetOpcode::G_BITCAST: 2174 return lowerBitcast(MI); 2175 case TargetOpcode::G_SREM: 2176 case TargetOpcode::G_UREM: { 2177 auto Quot = 2178 MIRBuilder.buildInstr(MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV, {Ty}, 2179 {MI.getOperand(1), MI.getOperand(2)}); 2180 2181 auto Prod = MIRBuilder.buildMul(Ty, Quot, MI.getOperand(2)); 2182 MIRBuilder.buildSub(MI.getOperand(0), MI.getOperand(1), Prod); 2183 MI.eraseFromParent(); 2184 return Legalized; 2185 } 2186 case TargetOpcode::G_SADDO: 2187 case TargetOpcode::G_SSUBO: 2188 return lowerSADDO_SSUBO(MI); 2189 case TargetOpcode::G_SMULO: 2190 case TargetOpcode::G_UMULO: { 2191 // Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the 2192 // result. 2193 Register Res = MI.getOperand(0).getReg(); 2194 Register Overflow = MI.getOperand(1).getReg(); 2195 Register LHS = MI.getOperand(2).getReg(); 2196 Register RHS = MI.getOperand(3).getReg(); 2197 2198 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO 2199 ? TargetOpcode::G_SMULH 2200 : TargetOpcode::G_UMULH; 2201 2202 Observer.changingInstr(MI); 2203 const auto &TII = MIRBuilder.getTII(); 2204 MI.setDesc(TII.get(TargetOpcode::G_MUL)); 2205 MI.RemoveOperand(1); 2206 Observer.changedInstr(MI); 2207 2208 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); 2209 2210 auto HiPart = MIRBuilder.buildInstr(Opcode, {Ty}, {LHS, RHS}); 2211 auto Zero = MIRBuilder.buildConstant(Ty, 0); 2212 2213 // For *signed* multiply, overflow is detected by checking: 2214 // (hi != (lo >> bitwidth-1)) 2215 if (Opcode == TargetOpcode::G_SMULH) { 2216 auto ShiftAmt = MIRBuilder.buildConstant(Ty, Ty.getSizeInBits() - 1); 2217 auto Shifted = MIRBuilder.buildAShr(Ty, Res, ShiftAmt); 2218 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Shifted); 2219 } else { 2220 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Zero); 2221 } 2222 return Legalized; 2223 } 2224 case TargetOpcode::G_FNEG: { 2225 // TODO: Handle vector types once we are able to 2226 // represent them. 2227 if (Ty.isVector()) 2228 return UnableToLegalize; 2229 Register Res = MI.getOperand(0).getReg(); 2230 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext(); 2231 Type *ZeroTy = getFloatTypeForLLT(Ctx, Ty); 2232 if (!ZeroTy) 2233 return UnableToLegalize; 2234 ConstantFP &ZeroForNegation = 2235 *cast<ConstantFP>(ConstantFP::getZeroValueForNegation(ZeroTy)); 2236 auto Zero = MIRBuilder.buildFConstant(Ty, ZeroForNegation); 2237 Register SubByReg = MI.getOperand(1).getReg(); 2238 Register ZeroReg = Zero.getReg(0); 2239 MIRBuilder.buildFSub(Res, ZeroReg, SubByReg, MI.getFlags()); 2240 MI.eraseFromParent(); 2241 return Legalized; 2242 } 2243 case TargetOpcode::G_FSUB: { 2244 // Lower (G_FSUB LHS, RHS) to (G_FADD LHS, (G_FNEG RHS)). 2245 // First, check if G_FNEG is marked as Lower. If so, we may 2246 // end up with an infinite loop as G_FSUB is used to legalize G_FNEG. 2247 if (LI.getAction({G_FNEG, {Ty}}).Action == Lower) 2248 return UnableToLegalize; 2249 Register Res = MI.getOperand(0).getReg(); 2250 Register LHS = MI.getOperand(1).getReg(); 2251 Register RHS = MI.getOperand(2).getReg(); 2252 Register Neg = MRI.createGenericVirtualRegister(Ty); 2253 MIRBuilder.buildFNeg(Neg, RHS); 2254 MIRBuilder.buildFAdd(Res, LHS, Neg, MI.getFlags()); 2255 MI.eraseFromParent(); 2256 return Legalized; 2257 } 2258 case TargetOpcode::G_FMAD: 2259 return lowerFMad(MI); 2260 case TargetOpcode::G_INTRINSIC_ROUND: 2261 return lowerIntrinsicRound(MI); 2262 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: { 2263 Register OldValRes = MI.getOperand(0).getReg(); 2264 Register SuccessRes = MI.getOperand(1).getReg(); 2265 Register Addr = MI.getOperand(2).getReg(); 2266 Register CmpVal = MI.getOperand(3).getReg(); 2267 Register NewVal = MI.getOperand(4).getReg(); 2268 MIRBuilder.buildAtomicCmpXchg(OldValRes, Addr, CmpVal, NewVal, 2269 **MI.memoperands_begin()); 2270 MIRBuilder.buildICmp(CmpInst::ICMP_EQ, SuccessRes, OldValRes, CmpVal); 2271 MI.eraseFromParent(); 2272 return Legalized; 2273 } 2274 case TargetOpcode::G_LOAD: 2275 case TargetOpcode::G_SEXTLOAD: 2276 case TargetOpcode::G_ZEXTLOAD: { 2277 // Lower to a memory-width G_LOAD and a G_SEXT/G_ZEXT/G_ANYEXT 2278 Register DstReg = MI.getOperand(0).getReg(); 2279 Register PtrReg = MI.getOperand(1).getReg(); 2280 LLT DstTy = MRI.getType(DstReg); 2281 auto &MMO = **MI.memoperands_begin(); 2282 2283 if (DstTy.getSizeInBits() == MMO.getSizeInBits()) { 2284 if (MI.getOpcode() == TargetOpcode::G_LOAD) { 2285 // This load needs splitting into power of 2 sized loads. 2286 if (DstTy.isVector()) 2287 return UnableToLegalize; 2288 if (isPowerOf2_32(DstTy.getSizeInBits())) 2289 return UnableToLegalize; // Don't know what we're being asked to do. 2290 2291 // Our strategy here is to generate anyextending loads for the smaller 2292 // types up to next power-2 result type, and then combine the two larger 2293 // result values together, before truncating back down to the non-pow-2 2294 // type. 2295 // E.g. v1 = i24 load => 2296 // v2 = i32 zextload (2 byte) 2297 // v3 = i32 load (1 byte) 2298 // v4 = i32 shl v3, 16 2299 // v5 = i32 or v4, v2 2300 // v1 = i24 trunc v5 2301 // By doing this we generate the correct truncate which should get 2302 // combined away as an artifact with a matching extend. 2303 uint64_t LargeSplitSize = PowerOf2Floor(DstTy.getSizeInBits()); 2304 uint64_t SmallSplitSize = DstTy.getSizeInBits() - LargeSplitSize; 2305 2306 MachineFunction &MF = MIRBuilder.getMF(); 2307 MachineMemOperand *LargeMMO = 2308 MF.getMachineMemOperand(&MMO, 0, LargeSplitSize / 8); 2309 MachineMemOperand *SmallMMO = MF.getMachineMemOperand( 2310 &MMO, LargeSplitSize / 8, SmallSplitSize / 8); 2311 2312 LLT PtrTy = MRI.getType(PtrReg); 2313 unsigned AnyExtSize = NextPowerOf2(DstTy.getSizeInBits()); 2314 LLT AnyExtTy = LLT::scalar(AnyExtSize); 2315 Register LargeLdReg = MRI.createGenericVirtualRegister(AnyExtTy); 2316 Register SmallLdReg = MRI.createGenericVirtualRegister(AnyExtTy); 2317 auto LargeLoad = MIRBuilder.buildLoadInstr( 2318 TargetOpcode::G_ZEXTLOAD, LargeLdReg, PtrReg, *LargeMMO); 2319 2320 auto OffsetCst = MIRBuilder.buildConstant( 2321 LLT::scalar(PtrTy.getSizeInBits()), LargeSplitSize / 8); 2322 Register PtrAddReg = MRI.createGenericVirtualRegister(PtrTy); 2323 auto SmallPtr = 2324 MIRBuilder.buildPtrAdd(PtrAddReg, PtrReg, OffsetCst.getReg(0)); 2325 auto SmallLoad = MIRBuilder.buildLoad(SmallLdReg, SmallPtr.getReg(0), 2326 *SmallMMO); 2327 2328 auto ShiftAmt = MIRBuilder.buildConstant(AnyExtTy, LargeSplitSize); 2329 auto Shift = MIRBuilder.buildShl(AnyExtTy, SmallLoad, ShiftAmt); 2330 auto Or = MIRBuilder.buildOr(AnyExtTy, Shift, LargeLoad); 2331 MIRBuilder.buildTrunc(DstReg, {Or.getReg(0)}); 2332 MI.eraseFromParent(); 2333 return Legalized; 2334 } 2335 MIRBuilder.buildLoad(DstReg, PtrReg, MMO); 2336 MI.eraseFromParent(); 2337 return Legalized; 2338 } 2339 2340 if (DstTy.isScalar()) { 2341 Register TmpReg = 2342 MRI.createGenericVirtualRegister(LLT::scalar(MMO.getSizeInBits())); 2343 MIRBuilder.buildLoad(TmpReg, PtrReg, MMO); 2344 switch (MI.getOpcode()) { 2345 default: 2346 llvm_unreachable("Unexpected opcode"); 2347 case TargetOpcode::G_LOAD: 2348 MIRBuilder.buildExtOrTrunc(TargetOpcode::G_ANYEXT, DstReg, TmpReg); 2349 break; 2350 case TargetOpcode::G_SEXTLOAD: 2351 MIRBuilder.buildSExt(DstReg, TmpReg); 2352 break; 2353 case TargetOpcode::G_ZEXTLOAD: 2354 MIRBuilder.buildZExt(DstReg, TmpReg); 2355 break; 2356 } 2357 MI.eraseFromParent(); 2358 return Legalized; 2359 } 2360 2361 return UnableToLegalize; 2362 } 2363 case TargetOpcode::G_STORE: { 2364 // Lower a non-power of 2 store into multiple pow-2 stores. 2365 // E.g. split an i24 store into an i16 store + i8 store. 2366 // We do this by first extending the stored value to the next largest power 2367 // of 2 type, and then using truncating stores to store the components. 2368 // By doing this, likewise with G_LOAD, generate an extend that can be 2369 // artifact-combined away instead of leaving behind extracts. 2370 Register SrcReg = MI.getOperand(0).getReg(); 2371 Register PtrReg = MI.getOperand(1).getReg(); 2372 LLT SrcTy = MRI.getType(SrcReg); 2373 MachineMemOperand &MMO = **MI.memoperands_begin(); 2374 if (SrcTy.getSizeInBits() != MMO.getSizeInBits()) 2375 return UnableToLegalize; 2376 if (SrcTy.isVector()) 2377 return UnableToLegalize; 2378 if (isPowerOf2_32(SrcTy.getSizeInBits())) 2379 return UnableToLegalize; // Don't know what we're being asked to do. 2380 2381 // Extend to the next pow-2. 2382 const LLT ExtendTy = LLT::scalar(NextPowerOf2(SrcTy.getSizeInBits())); 2383 auto ExtVal = MIRBuilder.buildAnyExt(ExtendTy, SrcReg); 2384 2385 // Obtain the smaller value by shifting away the larger value. 2386 uint64_t LargeSplitSize = PowerOf2Floor(SrcTy.getSizeInBits()); 2387 uint64_t SmallSplitSize = SrcTy.getSizeInBits() - LargeSplitSize; 2388 auto ShiftAmt = MIRBuilder.buildConstant(ExtendTy, LargeSplitSize); 2389 auto SmallVal = MIRBuilder.buildLShr(ExtendTy, ExtVal, ShiftAmt); 2390 2391 // Generate the PtrAdd and truncating stores. 2392 LLT PtrTy = MRI.getType(PtrReg); 2393 auto OffsetCst = MIRBuilder.buildConstant( 2394 LLT::scalar(PtrTy.getSizeInBits()), LargeSplitSize / 8); 2395 Register PtrAddReg = MRI.createGenericVirtualRegister(PtrTy); 2396 auto SmallPtr = 2397 MIRBuilder.buildPtrAdd(PtrAddReg, PtrReg, OffsetCst.getReg(0)); 2398 2399 MachineFunction &MF = MIRBuilder.getMF(); 2400 MachineMemOperand *LargeMMO = 2401 MF.getMachineMemOperand(&MMO, 0, LargeSplitSize / 8); 2402 MachineMemOperand *SmallMMO = 2403 MF.getMachineMemOperand(&MMO, LargeSplitSize / 8, SmallSplitSize / 8); 2404 MIRBuilder.buildStore(ExtVal.getReg(0), PtrReg, *LargeMMO); 2405 MIRBuilder.buildStore(SmallVal.getReg(0), SmallPtr.getReg(0), *SmallMMO); 2406 MI.eraseFromParent(); 2407 return Legalized; 2408 } 2409 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 2410 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 2411 case TargetOpcode::G_CTLZ: 2412 case TargetOpcode::G_CTTZ: 2413 case TargetOpcode::G_CTPOP: 2414 return lowerBitCount(MI, TypeIdx, Ty); 2415 case G_UADDO: { 2416 Register Res = MI.getOperand(0).getReg(); 2417 Register CarryOut = MI.getOperand(1).getReg(); 2418 Register LHS = MI.getOperand(2).getReg(); 2419 Register RHS = MI.getOperand(3).getReg(); 2420 2421 MIRBuilder.buildAdd(Res, LHS, RHS); 2422 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, RHS); 2423 2424 MI.eraseFromParent(); 2425 return Legalized; 2426 } 2427 case G_UADDE: { 2428 Register Res = MI.getOperand(0).getReg(); 2429 Register CarryOut = MI.getOperand(1).getReg(); 2430 Register LHS = MI.getOperand(2).getReg(); 2431 Register RHS = MI.getOperand(3).getReg(); 2432 Register CarryIn = MI.getOperand(4).getReg(); 2433 LLT Ty = MRI.getType(Res); 2434 2435 auto TmpRes = MIRBuilder.buildAdd(Ty, LHS, RHS); 2436 auto ZExtCarryIn = MIRBuilder.buildZExt(Ty, CarryIn); 2437 MIRBuilder.buildAdd(Res, TmpRes, ZExtCarryIn); 2438 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, LHS); 2439 2440 MI.eraseFromParent(); 2441 return Legalized; 2442 } 2443 case G_USUBO: { 2444 Register Res = MI.getOperand(0).getReg(); 2445 Register BorrowOut = MI.getOperand(1).getReg(); 2446 Register LHS = MI.getOperand(2).getReg(); 2447 Register RHS = MI.getOperand(3).getReg(); 2448 2449 MIRBuilder.buildSub(Res, LHS, RHS); 2450 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, BorrowOut, LHS, RHS); 2451 2452 MI.eraseFromParent(); 2453 return Legalized; 2454 } 2455 case G_USUBE: { 2456 Register Res = MI.getOperand(0).getReg(); 2457 Register BorrowOut = MI.getOperand(1).getReg(); 2458 Register LHS = MI.getOperand(2).getReg(); 2459 Register RHS = MI.getOperand(3).getReg(); 2460 Register BorrowIn = MI.getOperand(4).getReg(); 2461 const LLT CondTy = MRI.getType(BorrowOut); 2462 const LLT Ty = MRI.getType(Res); 2463 2464 auto TmpRes = MIRBuilder.buildSub(Ty, LHS, RHS); 2465 auto ZExtBorrowIn = MIRBuilder.buildZExt(Ty, BorrowIn); 2466 MIRBuilder.buildSub(Res, TmpRes, ZExtBorrowIn); 2467 2468 auto LHS_EQ_RHS = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, CondTy, LHS, RHS); 2469 auto LHS_ULT_RHS = MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CondTy, LHS, RHS); 2470 MIRBuilder.buildSelect(BorrowOut, LHS_EQ_RHS, BorrowIn, LHS_ULT_RHS); 2471 2472 MI.eraseFromParent(); 2473 return Legalized; 2474 } 2475 case G_UITOFP: 2476 return lowerUITOFP(MI, TypeIdx, Ty); 2477 case G_SITOFP: 2478 return lowerSITOFP(MI, TypeIdx, Ty); 2479 case G_FPTOUI: 2480 return lowerFPTOUI(MI, TypeIdx, Ty); 2481 case G_FPTOSI: 2482 return lowerFPTOSI(MI); 2483 case G_FPTRUNC: 2484 return lowerFPTRUNC(MI, TypeIdx, Ty); 2485 case G_SMIN: 2486 case G_SMAX: 2487 case G_UMIN: 2488 case G_UMAX: 2489 return lowerMinMax(MI, TypeIdx, Ty); 2490 case G_FCOPYSIGN: 2491 return lowerFCopySign(MI, TypeIdx, Ty); 2492 case G_FMINNUM: 2493 case G_FMAXNUM: 2494 return lowerFMinNumMaxNum(MI); 2495 case G_UNMERGE_VALUES: 2496 return lowerUnmergeValues(MI); 2497 case TargetOpcode::G_SEXT_INREG: { 2498 assert(MI.getOperand(2).isImm() && "Expected immediate"); 2499 int64_t SizeInBits = MI.getOperand(2).getImm(); 2500 2501 Register DstReg = MI.getOperand(0).getReg(); 2502 Register SrcReg = MI.getOperand(1).getReg(); 2503 LLT DstTy = MRI.getType(DstReg); 2504 Register TmpRes = MRI.createGenericVirtualRegister(DstTy); 2505 2506 auto MIBSz = MIRBuilder.buildConstant(DstTy, DstTy.getScalarSizeInBits() - SizeInBits); 2507 MIRBuilder.buildShl(TmpRes, SrcReg, MIBSz->getOperand(0)); 2508 MIRBuilder.buildAShr(DstReg, TmpRes, MIBSz->getOperand(0)); 2509 MI.eraseFromParent(); 2510 return Legalized; 2511 } 2512 case G_SHUFFLE_VECTOR: 2513 return lowerShuffleVector(MI); 2514 case G_DYN_STACKALLOC: 2515 return lowerDynStackAlloc(MI); 2516 case G_EXTRACT: 2517 return lowerExtract(MI); 2518 case G_INSERT: 2519 return lowerInsert(MI); 2520 case G_BSWAP: 2521 return lowerBswap(MI); 2522 case G_BITREVERSE: 2523 return lowerBitreverse(MI); 2524 case G_READ_REGISTER: 2525 case G_WRITE_REGISTER: 2526 return lowerReadWriteRegister(MI); 2527 } 2528 } 2529 2530 LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorImplicitDef( 2531 MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) { 2532 SmallVector<Register, 2> DstRegs; 2533 2534 unsigned NarrowSize = NarrowTy.getSizeInBits(); 2535 Register DstReg = MI.getOperand(0).getReg(); 2536 unsigned Size = MRI.getType(DstReg).getSizeInBits(); 2537 int NumParts = Size / NarrowSize; 2538 // FIXME: Don't know how to handle the situation where the small vectors 2539 // aren't all the same size yet. 2540 if (Size % NarrowSize != 0) 2541 return UnableToLegalize; 2542 2543 for (int i = 0; i < NumParts; ++i) { 2544 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy); 2545 MIRBuilder.buildUndef(TmpReg); 2546 DstRegs.push_back(TmpReg); 2547 } 2548 2549 if (NarrowTy.isVector()) 2550 MIRBuilder.buildConcatVectors(DstReg, DstRegs); 2551 else 2552 MIRBuilder.buildBuildVector(DstReg, DstRegs); 2553 2554 MI.eraseFromParent(); 2555 return Legalized; 2556 } 2557 2558 LegalizerHelper::LegalizeResult 2559 LegalizerHelper::fewerElementsVectorBasic(MachineInstr &MI, unsigned TypeIdx, 2560 LLT NarrowTy) { 2561 assert(TypeIdx == 0 && "only one type index expected"); 2562 2563 const unsigned Opc = MI.getOpcode(); 2564 const int NumOps = MI.getNumOperands() - 1; 2565 const Register DstReg = MI.getOperand(0).getReg(); 2566 const unsigned Flags = MI.getFlags(); 2567 2568 assert(NumOps <= 3 && "expected instrution with 1 result and 1-3 sources"); 2569 2570 SmallVector<Register, 8> ExtractedRegs[3]; 2571 SmallVector<Register, 8> Parts; 2572 2573 // Break down all the sources into NarrowTy pieces we can operate on. This may 2574 // involve creating merges to a wider type, padded with undef. 2575 for (int I = 0; I != NumOps; ++I) { 2576 Register SrcReg = MI.getOperand(I + 1).getReg(); 2577 LLT SrcTy = MRI.getType(SrcReg); 2578 LLT GCDTy = extractGCDType(ExtractedRegs[I], SrcTy, NarrowTy, SrcReg); 2579 2580 // Build a sequence of NarrowTy pieces in ExtractedRegs for this operand. 2581 buildLCMMergePieces(SrcTy, NarrowTy, GCDTy, ExtractedRegs[I], 2582 TargetOpcode::G_ANYEXT); 2583 } 2584 2585 SmallVector<Register, 8> ResultRegs; 2586 2587 // Input operands for each sub-instruction. 2588 SmallVector<SrcOp, 4> InputRegs(NumOps, Register()); 2589 2590 int NumParts = ExtractedRegs[0].size(); 2591 const unsigned DstSize = MRI.getType(DstReg).getSizeInBits(); 2592 const unsigned NarrowSize = NarrowTy.getSizeInBits(); 2593 2594 // We widened the source registers to satisfy merge/unmerge size 2595 // constraints. We'll have some extra fully undef parts. 2596 const int NumRealParts = (DstSize + NarrowSize - 1) / NarrowSize; 2597 2598 for (int I = 0; I != NumRealParts; ++I) { 2599 // Emit this instruction on each of the split pieces. 2600 for (int J = 0; J != NumOps; ++J) 2601 InputRegs[J] = ExtractedRegs[J][I]; 2602 2603 auto Inst = MIRBuilder.buildInstr(Opc, {NarrowTy}, InputRegs, Flags); 2604 ResultRegs.push_back(Inst.getReg(0)); 2605 } 2606 2607 // Fill out the widened result with undef instead of creating instructions 2608 // with undef inputs. 2609 int NumUndefParts = NumParts - NumRealParts; 2610 if (NumUndefParts != 0) 2611 ResultRegs.append(NumUndefParts, MIRBuilder.buildUndef(NarrowTy).getReg(0)); 2612 2613 // Extract the possibly padded result to the original result register. 2614 LLT DstTy = MRI.getType(DstReg); 2615 LLT LCMTy = getLCMType(DstTy, NarrowTy); 2616 buildWidenedRemergeToDst(DstReg, LCMTy, ResultRegs); 2617 2618 MI.eraseFromParent(); 2619 return Legalized; 2620 } 2621 2622 // Handle splitting vector operations which need to have the same number of 2623 // elements in each type index, but each type index may have a different element 2624 // type. 2625 // 2626 // e.g. <4 x s64> = G_SHL <4 x s64>, <4 x s32> -> 2627 // <2 x s64> = G_SHL <2 x s64>, <2 x s32> 2628 // <2 x s64> = G_SHL <2 x s64>, <2 x s32> 2629 // 2630 // Also handles some irregular breakdown cases, e.g. 2631 // e.g. <3 x s64> = G_SHL <3 x s64>, <3 x s32> -> 2632 // <2 x s64> = G_SHL <2 x s64>, <2 x s32> 2633 // s64 = G_SHL s64, s32 2634 LegalizerHelper::LegalizeResult 2635 LegalizerHelper::fewerElementsVectorMultiEltType( 2636 MachineInstr &MI, unsigned TypeIdx, LLT NarrowTyArg) { 2637 if (TypeIdx != 0) 2638 return UnableToLegalize; 2639 2640 const LLT NarrowTy0 = NarrowTyArg; 2641 const unsigned NewNumElts = 2642 NarrowTy0.isVector() ? NarrowTy0.getNumElements() : 1; 2643 2644 const Register DstReg = MI.getOperand(0).getReg(); 2645 LLT DstTy = MRI.getType(DstReg); 2646 LLT LeftoverTy0; 2647 2648 // All of the operands need to have the same number of elements, so if we can 2649 // determine a type breakdown for the result type, we can for all of the 2650 // source types. 2651 int NumParts = getNarrowTypeBreakDown(DstTy, NarrowTy0, LeftoverTy0).first; 2652 if (NumParts < 0) 2653 return UnableToLegalize; 2654 2655 SmallVector<MachineInstrBuilder, 4> NewInsts; 2656 2657 SmallVector<Register, 4> DstRegs, LeftoverDstRegs; 2658 SmallVector<Register, 4> PartRegs, LeftoverRegs; 2659 2660 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) { 2661 LLT LeftoverTy; 2662 Register SrcReg = MI.getOperand(I).getReg(); 2663 LLT SrcTyI = MRI.getType(SrcReg); 2664 LLT NarrowTyI = LLT::scalarOrVector(NewNumElts, SrcTyI.getScalarType()); 2665 LLT LeftoverTyI; 2666 2667 // Split this operand into the requested typed registers, and any leftover 2668 // required to reproduce the original type. 2669 if (!extractParts(SrcReg, SrcTyI, NarrowTyI, LeftoverTyI, PartRegs, 2670 LeftoverRegs)) 2671 return UnableToLegalize; 2672 2673 if (I == 1) { 2674 // For the first operand, create an instruction for each part and setup 2675 // the result. 2676 for (Register PartReg : PartRegs) { 2677 Register PartDstReg = MRI.createGenericVirtualRegister(NarrowTy0); 2678 NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode()) 2679 .addDef(PartDstReg) 2680 .addUse(PartReg)); 2681 DstRegs.push_back(PartDstReg); 2682 } 2683 2684 for (Register LeftoverReg : LeftoverRegs) { 2685 Register PartDstReg = MRI.createGenericVirtualRegister(LeftoverTy0); 2686 NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode()) 2687 .addDef(PartDstReg) 2688 .addUse(LeftoverReg)); 2689 LeftoverDstRegs.push_back(PartDstReg); 2690 } 2691 } else { 2692 assert(NewInsts.size() == PartRegs.size() + LeftoverRegs.size()); 2693 2694 // Add the newly created operand splits to the existing instructions. The 2695 // odd-sized pieces are ordered after the requested NarrowTyArg sized 2696 // pieces. 2697 unsigned InstCount = 0; 2698 for (unsigned J = 0, JE = PartRegs.size(); J != JE; ++J) 2699 NewInsts[InstCount++].addUse(PartRegs[J]); 2700 for (unsigned J = 0, JE = LeftoverRegs.size(); J != JE; ++J) 2701 NewInsts[InstCount++].addUse(LeftoverRegs[J]); 2702 } 2703 2704 PartRegs.clear(); 2705 LeftoverRegs.clear(); 2706 } 2707 2708 // Insert the newly built operations and rebuild the result register. 2709 for (auto &MIB : NewInsts) 2710 MIRBuilder.insertInstr(MIB); 2711 2712 insertParts(DstReg, DstTy, NarrowTy0, DstRegs, LeftoverTy0, LeftoverDstRegs); 2713 2714 MI.eraseFromParent(); 2715 return Legalized; 2716 } 2717 2718 LegalizerHelper::LegalizeResult 2719 LegalizerHelper::fewerElementsVectorCasts(MachineInstr &MI, unsigned TypeIdx, 2720 LLT NarrowTy) { 2721 if (TypeIdx != 0) 2722 return UnableToLegalize; 2723 2724 Register DstReg = MI.getOperand(0).getReg(); 2725 Register SrcReg = MI.getOperand(1).getReg(); 2726 LLT DstTy = MRI.getType(DstReg); 2727 LLT SrcTy = MRI.getType(SrcReg); 2728 2729 LLT NarrowTy0 = NarrowTy; 2730 LLT NarrowTy1; 2731 unsigned NumParts; 2732 2733 if (NarrowTy.isVector()) { 2734 // Uneven breakdown not handled. 2735 NumParts = DstTy.getNumElements() / NarrowTy.getNumElements(); 2736 if (NumParts * NarrowTy.getNumElements() != DstTy.getNumElements()) 2737 return UnableToLegalize; 2738 2739 NarrowTy1 = LLT::vector(NumParts, SrcTy.getElementType().getSizeInBits()); 2740 } else { 2741 NumParts = DstTy.getNumElements(); 2742 NarrowTy1 = SrcTy.getElementType(); 2743 } 2744 2745 SmallVector<Register, 4> SrcRegs, DstRegs; 2746 extractParts(SrcReg, NarrowTy1, NumParts, SrcRegs); 2747 2748 for (unsigned I = 0; I < NumParts; ++I) { 2749 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0); 2750 MachineInstr *NewInst = 2751 MIRBuilder.buildInstr(MI.getOpcode(), {DstReg}, {SrcRegs[I]}); 2752 2753 NewInst->setFlags(MI.getFlags()); 2754 DstRegs.push_back(DstReg); 2755 } 2756 2757 if (NarrowTy.isVector()) 2758 MIRBuilder.buildConcatVectors(DstReg, DstRegs); 2759 else 2760 MIRBuilder.buildBuildVector(DstReg, DstRegs); 2761 2762 MI.eraseFromParent(); 2763 return Legalized; 2764 } 2765 2766 LegalizerHelper::LegalizeResult 2767 LegalizerHelper::fewerElementsVectorCmp(MachineInstr &MI, unsigned TypeIdx, 2768 LLT NarrowTy) { 2769 Register DstReg = MI.getOperand(0).getReg(); 2770 Register Src0Reg = MI.getOperand(2).getReg(); 2771 LLT DstTy = MRI.getType(DstReg); 2772 LLT SrcTy = MRI.getType(Src0Reg); 2773 2774 unsigned NumParts; 2775 LLT NarrowTy0, NarrowTy1; 2776 2777 if (TypeIdx == 0) { 2778 unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1; 2779 unsigned OldElts = DstTy.getNumElements(); 2780 2781 NarrowTy0 = NarrowTy; 2782 NumParts = NarrowTy.isVector() ? (OldElts / NewElts) : DstTy.getNumElements(); 2783 NarrowTy1 = NarrowTy.isVector() ? 2784 LLT::vector(NarrowTy.getNumElements(), SrcTy.getScalarSizeInBits()) : 2785 SrcTy.getElementType(); 2786 2787 } else { 2788 unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1; 2789 unsigned OldElts = SrcTy.getNumElements(); 2790 2791 NumParts = NarrowTy.isVector() ? (OldElts / NewElts) : 2792 NarrowTy.getNumElements(); 2793 NarrowTy0 = LLT::vector(NarrowTy.getNumElements(), 2794 DstTy.getScalarSizeInBits()); 2795 NarrowTy1 = NarrowTy; 2796 } 2797 2798 // FIXME: Don't know how to handle the situation where the small vectors 2799 // aren't all the same size yet. 2800 if (NarrowTy1.isVector() && 2801 NarrowTy1.getNumElements() * NumParts != DstTy.getNumElements()) 2802 return UnableToLegalize; 2803 2804 CmpInst::Predicate Pred 2805 = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate()); 2806 2807 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs; 2808 extractParts(MI.getOperand(2).getReg(), NarrowTy1, NumParts, Src1Regs); 2809 extractParts(MI.getOperand(3).getReg(), NarrowTy1, NumParts, Src2Regs); 2810 2811 for (unsigned I = 0; I < NumParts; ++I) { 2812 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0); 2813 DstRegs.push_back(DstReg); 2814 2815 if (MI.getOpcode() == TargetOpcode::G_ICMP) 2816 MIRBuilder.buildICmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]); 2817 else { 2818 MachineInstr *NewCmp 2819 = MIRBuilder.buildFCmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]); 2820 NewCmp->setFlags(MI.getFlags()); 2821 } 2822 } 2823 2824 if (NarrowTy1.isVector()) 2825 MIRBuilder.buildConcatVectors(DstReg, DstRegs); 2826 else 2827 MIRBuilder.buildBuildVector(DstReg, DstRegs); 2828 2829 MI.eraseFromParent(); 2830 return Legalized; 2831 } 2832 2833 LegalizerHelper::LegalizeResult 2834 LegalizerHelper::fewerElementsVectorSelect(MachineInstr &MI, unsigned TypeIdx, 2835 LLT NarrowTy) { 2836 Register DstReg = MI.getOperand(0).getReg(); 2837 Register CondReg = MI.getOperand(1).getReg(); 2838 2839 unsigned NumParts = 0; 2840 LLT NarrowTy0, NarrowTy1; 2841 2842 LLT DstTy = MRI.getType(DstReg); 2843 LLT CondTy = MRI.getType(CondReg); 2844 unsigned Size = DstTy.getSizeInBits(); 2845 2846 assert(TypeIdx == 0 || CondTy.isVector()); 2847 2848 if (TypeIdx == 0) { 2849 NarrowTy0 = NarrowTy; 2850 NarrowTy1 = CondTy; 2851 2852 unsigned NarrowSize = NarrowTy0.getSizeInBits(); 2853 // FIXME: Don't know how to handle the situation where the small vectors 2854 // aren't all the same size yet. 2855 if (Size % NarrowSize != 0) 2856 return UnableToLegalize; 2857 2858 NumParts = Size / NarrowSize; 2859 2860 // Need to break down the condition type 2861 if (CondTy.isVector()) { 2862 if (CondTy.getNumElements() == NumParts) 2863 NarrowTy1 = CondTy.getElementType(); 2864 else 2865 NarrowTy1 = LLT::vector(CondTy.getNumElements() / NumParts, 2866 CondTy.getScalarSizeInBits()); 2867 } 2868 } else { 2869 NumParts = CondTy.getNumElements(); 2870 if (NarrowTy.isVector()) { 2871 // TODO: Handle uneven breakdown. 2872 if (NumParts * NarrowTy.getNumElements() != CondTy.getNumElements()) 2873 return UnableToLegalize; 2874 2875 return UnableToLegalize; 2876 } else { 2877 NarrowTy0 = DstTy.getElementType(); 2878 NarrowTy1 = NarrowTy; 2879 } 2880 } 2881 2882 SmallVector<Register, 2> DstRegs, Src0Regs, Src1Regs, Src2Regs; 2883 if (CondTy.isVector()) 2884 extractParts(MI.getOperand(1).getReg(), NarrowTy1, NumParts, Src0Regs); 2885 2886 extractParts(MI.getOperand(2).getReg(), NarrowTy0, NumParts, Src1Regs); 2887 extractParts(MI.getOperand(3).getReg(), NarrowTy0, NumParts, Src2Regs); 2888 2889 for (unsigned i = 0; i < NumParts; ++i) { 2890 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0); 2891 MIRBuilder.buildSelect(DstReg, CondTy.isVector() ? Src0Regs[i] : CondReg, 2892 Src1Regs[i], Src2Regs[i]); 2893 DstRegs.push_back(DstReg); 2894 } 2895 2896 if (NarrowTy0.isVector()) 2897 MIRBuilder.buildConcatVectors(DstReg, DstRegs); 2898 else 2899 MIRBuilder.buildBuildVector(DstReg, DstRegs); 2900 2901 MI.eraseFromParent(); 2902 return Legalized; 2903 } 2904 2905 LegalizerHelper::LegalizeResult 2906 LegalizerHelper::fewerElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx, 2907 LLT NarrowTy) { 2908 const Register DstReg = MI.getOperand(0).getReg(); 2909 LLT PhiTy = MRI.getType(DstReg); 2910 LLT LeftoverTy; 2911 2912 // All of the operands need to have the same number of elements, so if we can 2913 // determine a type breakdown for the result type, we can for all of the 2914 // source types. 2915 int NumParts, NumLeftover; 2916 std::tie(NumParts, NumLeftover) 2917 = getNarrowTypeBreakDown(PhiTy, NarrowTy, LeftoverTy); 2918 if (NumParts < 0) 2919 return UnableToLegalize; 2920 2921 SmallVector<Register, 4> DstRegs, LeftoverDstRegs; 2922 SmallVector<MachineInstrBuilder, 4> NewInsts; 2923 2924 const int TotalNumParts = NumParts + NumLeftover; 2925 2926 // Insert the new phis in the result block first. 2927 for (int I = 0; I != TotalNumParts; ++I) { 2928 LLT Ty = I < NumParts ? NarrowTy : LeftoverTy; 2929 Register PartDstReg = MRI.createGenericVirtualRegister(Ty); 2930 NewInsts.push_back(MIRBuilder.buildInstr(TargetOpcode::G_PHI) 2931 .addDef(PartDstReg)); 2932 if (I < NumParts) 2933 DstRegs.push_back(PartDstReg); 2934 else 2935 LeftoverDstRegs.push_back(PartDstReg); 2936 } 2937 2938 MachineBasicBlock *MBB = MI.getParent(); 2939 MIRBuilder.setInsertPt(*MBB, MBB->getFirstNonPHI()); 2940 insertParts(DstReg, PhiTy, NarrowTy, DstRegs, LeftoverTy, LeftoverDstRegs); 2941 2942 SmallVector<Register, 4> PartRegs, LeftoverRegs; 2943 2944 // Insert code to extract the incoming values in each predecessor block. 2945 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) { 2946 PartRegs.clear(); 2947 LeftoverRegs.clear(); 2948 2949 Register SrcReg = MI.getOperand(I).getReg(); 2950 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB(); 2951 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator()); 2952 2953 LLT Unused; 2954 if (!extractParts(SrcReg, PhiTy, NarrowTy, Unused, PartRegs, 2955 LeftoverRegs)) 2956 return UnableToLegalize; 2957 2958 // Add the newly created operand splits to the existing instructions. The 2959 // odd-sized pieces are ordered after the requested NarrowTyArg sized 2960 // pieces. 2961 for (int J = 0; J != TotalNumParts; ++J) { 2962 MachineInstrBuilder MIB = NewInsts[J]; 2963 MIB.addUse(J < NumParts ? PartRegs[J] : LeftoverRegs[J - NumParts]); 2964 MIB.addMBB(&OpMBB); 2965 } 2966 } 2967 2968 MI.eraseFromParent(); 2969 return Legalized; 2970 } 2971 2972 LegalizerHelper::LegalizeResult 2973 LegalizerHelper::fewerElementsVectorUnmergeValues(MachineInstr &MI, 2974 unsigned TypeIdx, 2975 LLT NarrowTy) { 2976 if (TypeIdx != 1) 2977 return UnableToLegalize; 2978 2979 const int NumDst = MI.getNumOperands() - 1; 2980 const Register SrcReg = MI.getOperand(NumDst).getReg(); 2981 LLT SrcTy = MRI.getType(SrcReg); 2982 2983 LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); 2984 2985 // TODO: Create sequence of extracts. 2986 if (DstTy == NarrowTy) 2987 return UnableToLegalize; 2988 2989 LLT GCDTy = getGCDType(SrcTy, NarrowTy); 2990 if (DstTy == GCDTy) { 2991 // This would just be a copy of the same unmerge. 2992 // TODO: Create extracts, pad with undef and create intermediate merges. 2993 return UnableToLegalize; 2994 } 2995 2996 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg); 2997 const int NumUnmerge = Unmerge->getNumOperands() - 1; 2998 const int PartsPerUnmerge = NumDst / NumUnmerge; 2999 3000 for (int I = 0; I != NumUnmerge; ++I) { 3001 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES); 3002 3003 for (int J = 0; J != PartsPerUnmerge; ++J) 3004 MIB.addDef(MI.getOperand(I * PartsPerUnmerge + J).getReg()); 3005 MIB.addUse(Unmerge.getReg(I)); 3006 } 3007 3008 MI.eraseFromParent(); 3009 return Legalized; 3010 } 3011 3012 LegalizerHelper::LegalizeResult 3013 LegalizerHelper::fewerElementsVectorBuildVector(MachineInstr &MI, 3014 unsigned TypeIdx, 3015 LLT NarrowTy) { 3016 assert(TypeIdx == 0 && "not a vector type index"); 3017 Register DstReg = MI.getOperand(0).getReg(); 3018 LLT DstTy = MRI.getType(DstReg); 3019 LLT SrcTy = DstTy.getElementType(); 3020 3021 int DstNumElts = DstTy.getNumElements(); 3022 int NarrowNumElts = NarrowTy.getNumElements(); 3023 int NumConcat = (DstNumElts + NarrowNumElts - 1) / NarrowNumElts; 3024 LLT WidenedDstTy = LLT::vector(NarrowNumElts * NumConcat, SrcTy); 3025 3026 SmallVector<Register, 8> ConcatOps; 3027 SmallVector<Register, 8> SubBuildVector; 3028 3029 Register UndefReg; 3030 if (WidenedDstTy != DstTy) 3031 UndefReg = MIRBuilder.buildUndef(SrcTy).getReg(0); 3032 3033 // Create a G_CONCAT_VECTORS of NarrowTy pieces, padding with undef as 3034 // necessary. 3035 // 3036 // %3:_(<3 x s16>) = G_BUILD_VECTOR %0, %1, %2 3037 // -> <2 x s16> 3038 // 3039 // %4:_(s16) = G_IMPLICIT_DEF 3040 // %5:_(<2 x s16>) = G_BUILD_VECTOR %0, %1 3041 // %6:_(<2 x s16>) = G_BUILD_VECTOR %2, %4 3042 // %7:_(<4 x s16>) = G_CONCAT_VECTORS %5, %6 3043 // %3:_(<3 x s16>) = G_EXTRACT %7, 0 3044 for (int I = 0; I != NumConcat; ++I) { 3045 for (int J = 0; J != NarrowNumElts; ++J) { 3046 int SrcIdx = NarrowNumElts * I + J; 3047 3048 if (SrcIdx < DstNumElts) { 3049 Register SrcReg = MI.getOperand(SrcIdx + 1).getReg(); 3050 SubBuildVector.push_back(SrcReg); 3051 } else 3052 SubBuildVector.push_back(UndefReg); 3053 } 3054 3055 auto BuildVec = MIRBuilder.buildBuildVector(NarrowTy, SubBuildVector); 3056 ConcatOps.push_back(BuildVec.getReg(0)); 3057 SubBuildVector.clear(); 3058 } 3059 3060 if (DstTy == WidenedDstTy) 3061 MIRBuilder.buildConcatVectors(DstReg, ConcatOps); 3062 else { 3063 auto Concat = MIRBuilder.buildConcatVectors(WidenedDstTy, ConcatOps); 3064 MIRBuilder.buildExtract(DstReg, Concat, 0); 3065 } 3066 3067 MI.eraseFromParent(); 3068 return Legalized; 3069 } 3070 3071 LegalizerHelper::LegalizeResult 3072 LegalizerHelper::reduceLoadStoreWidth(MachineInstr &MI, unsigned TypeIdx, 3073 LLT NarrowTy) { 3074 // FIXME: Don't know how to handle secondary types yet. 3075 if (TypeIdx != 0) 3076 return UnableToLegalize; 3077 3078 MachineMemOperand *MMO = *MI.memoperands_begin(); 3079 3080 // This implementation doesn't work for atomics. Give up instead of doing 3081 // something invalid. 3082 if (MMO->getOrdering() != AtomicOrdering::NotAtomic || 3083 MMO->getFailureOrdering() != AtomicOrdering::NotAtomic) 3084 return UnableToLegalize; 3085 3086 bool IsLoad = MI.getOpcode() == TargetOpcode::G_LOAD; 3087 Register ValReg = MI.getOperand(0).getReg(); 3088 Register AddrReg = MI.getOperand(1).getReg(); 3089 LLT ValTy = MRI.getType(ValReg); 3090 3091 int NumParts = -1; 3092 int NumLeftover = -1; 3093 LLT LeftoverTy; 3094 SmallVector<Register, 8> NarrowRegs, NarrowLeftoverRegs; 3095 if (IsLoad) { 3096 std::tie(NumParts, NumLeftover) = getNarrowTypeBreakDown(ValTy, NarrowTy, LeftoverTy); 3097 } else { 3098 if (extractParts(ValReg, ValTy, NarrowTy, LeftoverTy, NarrowRegs, 3099 NarrowLeftoverRegs)) { 3100 NumParts = NarrowRegs.size(); 3101 NumLeftover = NarrowLeftoverRegs.size(); 3102 } 3103 } 3104 3105 if (NumParts == -1) 3106 return UnableToLegalize; 3107 3108 const LLT OffsetTy = LLT::scalar(MRI.getType(AddrReg).getScalarSizeInBits()); 3109 3110 unsigned TotalSize = ValTy.getSizeInBits(); 3111 3112 // Split the load/store into PartTy sized pieces starting at Offset. If this 3113 // is a load, return the new registers in ValRegs. For a store, each elements 3114 // of ValRegs should be PartTy. Returns the next offset that needs to be 3115 // handled. 3116 auto splitTypePieces = [=](LLT PartTy, SmallVectorImpl<Register> &ValRegs, 3117 unsigned Offset) -> unsigned { 3118 MachineFunction &MF = MIRBuilder.getMF(); 3119 unsigned PartSize = PartTy.getSizeInBits(); 3120 for (unsigned Idx = 0, E = NumParts; Idx != E && Offset < TotalSize; 3121 Offset += PartSize, ++Idx) { 3122 unsigned ByteSize = PartSize / 8; 3123 unsigned ByteOffset = Offset / 8; 3124 Register NewAddrReg; 3125 3126 MIRBuilder.materializePtrAdd(NewAddrReg, AddrReg, OffsetTy, ByteOffset); 3127 3128 MachineMemOperand *NewMMO = 3129 MF.getMachineMemOperand(MMO, ByteOffset, ByteSize); 3130 3131 if (IsLoad) { 3132 Register Dst = MRI.createGenericVirtualRegister(PartTy); 3133 ValRegs.push_back(Dst); 3134 MIRBuilder.buildLoad(Dst, NewAddrReg, *NewMMO); 3135 } else { 3136 MIRBuilder.buildStore(ValRegs[Idx], NewAddrReg, *NewMMO); 3137 } 3138 } 3139 3140 return Offset; 3141 }; 3142 3143 unsigned HandledOffset = splitTypePieces(NarrowTy, NarrowRegs, 0); 3144 3145 // Handle the rest of the register if this isn't an even type breakdown. 3146 if (LeftoverTy.isValid()) 3147 splitTypePieces(LeftoverTy, NarrowLeftoverRegs, HandledOffset); 3148 3149 if (IsLoad) { 3150 insertParts(ValReg, ValTy, NarrowTy, NarrowRegs, 3151 LeftoverTy, NarrowLeftoverRegs); 3152 } 3153 3154 MI.eraseFromParent(); 3155 return Legalized; 3156 } 3157 3158 LegalizerHelper::LegalizeResult 3159 LegalizerHelper::fewerElementsVectorSextInReg(MachineInstr &MI, unsigned TypeIdx, 3160 LLT NarrowTy) { 3161 Register DstReg = MI.getOperand(0).getReg(); 3162 Register SrcReg = MI.getOperand(1).getReg(); 3163 int64_t Imm = MI.getOperand(2).getImm(); 3164 3165 LLT DstTy = MRI.getType(DstReg); 3166 3167 SmallVector<Register, 8> Parts; 3168 LLT GCDTy = extractGCDType(Parts, DstTy, NarrowTy, SrcReg); 3169 LLT LCMTy = buildLCMMergePieces(DstTy, NarrowTy, GCDTy, Parts); 3170 3171 for (Register &R : Parts) 3172 R = MIRBuilder.buildSExtInReg(NarrowTy, R, Imm).getReg(0); 3173 3174 buildWidenedRemergeToDst(DstReg, LCMTy, Parts); 3175 3176 MI.eraseFromParent(); 3177 return Legalized; 3178 } 3179 3180 LegalizerHelper::LegalizeResult 3181 LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx, 3182 LLT NarrowTy) { 3183 using namespace TargetOpcode; 3184 3185 MIRBuilder.setInstr(MI); 3186 switch (MI.getOpcode()) { 3187 case G_IMPLICIT_DEF: 3188 return fewerElementsVectorImplicitDef(MI, TypeIdx, NarrowTy); 3189 case G_AND: 3190 case G_OR: 3191 case G_XOR: 3192 case G_ADD: 3193 case G_SUB: 3194 case G_MUL: 3195 case G_SMULH: 3196 case G_UMULH: 3197 case G_FADD: 3198 case G_FMUL: 3199 case G_FSUB: 3200 case G_FNEG: 3201 case G_FABS: 3202 case G_FCANONICALIZE: 3203 case G_FDIV: 3204 case G_FREM: 3205 case G_FMA: 3206 case G_FMAD: 3207 case G_FPOW: 3208 case G_FEXP: 3209 case G_FEXP2: 3210 case G_FLOG: 3211 case G_FLOG2: 3212 case G_FLOG10: 3213 case G_FNEARBYINT: 3214 case G_FCEIL: 3215 case G_FFLOOR: 3216 case G_FRINT: 3217 case G_INTRINSIC_ROUND: 3218 case G_INTRINSIC_TRUNC: 3219 case G_FCOS: 3220 case G_FSIN: 3221 case G_FSQRT: 3222 case G_BSWAP: 3223 case G_BITREVERSE: 3224 case G_SDIV: 3225 case G_UDIV: 3226 case G_SREM: 3227 case G_UREM: 3228 case G_SMIN: 3229 case G_SMAX: 3230 case G_UMIN: 3231 case G_UMAX: 3232 case G_FMINNUM: 3233 case G_FMAXNUM: 3234 case G_FMINNUM_IEEE: 3235 case G_FMAXNUM_IEEE: 3236 case G_FMINIMUM: 3237 case G_FMAXIMUM: 3238 return fewerElementsVectorBasic(MI, TypeIdx, NarrowTy); 3239 case G_SHL: 3240 case G_LSHR: 3241 case G_ASHR: 3242 case G_CTLZ: 3243 case G_CTLZ_ZERO_UNDEF: 3244 case G_CTTZ: 3245 case G_CTTZ_ZERO_UNDEF: 3246 case G_CTPOP: 3247 case G_FCOPYSIGN: 3248 return fewerElementsVectorMultiEltType(MI, TypeIdx, NarrowTy); 3249 case G_ZEXT: 3250 case G_SEXT: 3251 case G_ANYEXT: 3252 case G_FPEXT: 3253 case G_FPTRUNC: 3254 case G_SITOFP: 3255 case G_UITOFP: 3256 case G_FPTOSI: 3257 case G_FPTOUI: 3258 case G_INTTOPTR: 3259 case G_PTRTOINT: 3260 case G_ADDRSPACE_CAST: 3261 return fewerElementsVectorCasts(MI, TypeIdx, NarrowTy); 3262 case G_ICMP: 3263 case G_FCMP: 3264 return fewerElementsVectorCmp(MI, TypeIdx, NarrowTy); 3265 case G_SELECT: 3266 return fewerElementsVectorSelect(MI, TypeIdx, NarrowTy); 3267 case G_PHI: 3268 return fewerElementsVectorPhi(MI, TypeIdx, NarrowTy); 3269 case G_UNMERGE_VALUES: 3270 return fewerElementsVectorUnmergeValues(MI, TypeIdx, NarrowTy); 3271 case G_BUILD_VECTOR: 3272 return fewerElementsVectorBuildVector(MI, TypeIdx, NarrowTy); 3273 case G_LOAD: 3274 case G_STORE: 3275 return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy); 3276 case G_SEXT_INREG: 3277 return fewerElementsVectorSextInReg(MI, TypeIdx, NarrowTy); 3278 default: 3279 return UnableToLegalize; 3280 } 3281 } 3282 3283 LegalizerHelper::LegalizeResult 3284 LegalizerHelper::narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt, 3285 const LLT HalfTy, const LLT AmtTy) { 3286 3287 Register InL = MRI.createGenericVirtualRegister(HalfTy); 3288 Register InH = MRI.createGenericVirtualRegister(HalfTy); 3289 MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1)); 3290 3291 if (Amt.isNullValue()) { 3292 MIRBuilder.buildMerge(MI.getOperand(0), {InL, InH}); 3293 MI.eraseFromParent(); 3294 return Legalized; 3295 } 3296 3297 LLT NVT = HalfTy; 3298 unsigned NVTBits = HalfTy.getSizeInBits(); 3299 unsigned VTBits = 2 * NVTBits; 3300 3301 SrcOp Lo(Register(0)), Hi(Register(0)); 3302 if (MI.getOpcode() == TargetOpcode::G_SHL) { 3303 if (Amt.ugt(VTBits)) { 3304 Lo = Hi = MIRBuilder.buildConstant(NVT, 0); 3305 } else if (Amt.ugt(NVTBits)) { 3306 Lo = MIRBuilder.buildConstant(NVT, 0); 3307 Hi = MIRBuilder.buildShl(NVT, InL, 3308 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits)); 3309 } else if (Amt == NVTBits) { 3310 Lo = MIRBuilder.buildConstant(NVT, 0); 3311 Hi = InL; 3312 } else { 3313 Lo = MIRBuilder.buildShl(NVT, InL, MIRBuilder.buildConstant(AmtTy, Amt)); 3314 auto OrLHS = 3315 MIRBuilder.buildShl(NVT, InH, MIRBuilder.buildConstant(AmtTy, Amt)); 3316 auto OrRHS = MIRBuilder.buildLShr( 3317 NVT, InL, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits)); 3318 Hi = MIRBuilder.buildOr(NVT, OrLHS, OrRHS); 3319 } 3320 } else if (MI.getOpcode() == TargetOpcode::G_LSHR) { 3321 if (Amt.ugt(VTBits)) { 3322 Lo = Hi = MIRBuilder.buildConstant(NVT, 0); 3323 } else if (Amt.ugt(NVTBits)) { 3324 Lo = MIRBuilder.buildLShr(NVT, InH, 3325 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits)); 3326 Hi = MIRBuilder.buildConstant(NVT, 0); 3327 } else if (Amt == NVTBits) { 3328 Lo = InH; 3329 Hi = MIRBuilder.buildConstant(NVT, 0); 3330 } else { 3331 auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt); 3332 3333 auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst); 3334 auto OrRHS = MIRBuilder.buildShl( 3335 NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits)); 3336 3337 Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS); 3338 Hi = MIRBuilder.buildLShr(NVT, InH, ShiftAmtConst); 3339 } 3340 } else { 3341 if (Amt.ugt(VTBits)) { 3342 Hi = Lo = MIRBuilder.buildAShr( 3343 NVT, InH, MIRBuilder.buildConstant(AmtTy, NVTBits - 1)); 3344 } else if (Amt.ugt(NVTBits)) { 3345 Lo = MIRBuilder.buildAShr(NVT, InH, 3346 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits)); 3347 Hi = MIRBuilder.buildAShr(NVT, InH, 3348 MIRBuilder.buildConstant(AmtTy, NVTBits - 1)); 3349 } else if (Amt == NVTBits) { 3350 Lo = InH; 3351 Hi = MIRBuilder.buildAShr(NVT, InH, 3352 MIRBuilder.buildConstant(AmtTy, NVTBits - 1)); 3353 } else { 3354 auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt); 3355 3356 auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst); 3357 auto OrRHS = MIRBuilder.buildShl( 3358 NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits)); 3359 3360 Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS); 3361 Hi = MIRBuilder.buildAShr(NVT, InH, ShiftAmtConst); 3362 } 3363 } 3364 3365 MIRBuilder.buildMerge(MI.getOperand(0), {Lo, Hi}); 3366 MI.eraseFromParent(); 3367 3368 return Legalized; 3369 } 3370 3371 // TODO: Optimize if constant shift amount. 3372 LegalizerHelper::LegalizeResult 3373 LegalizerHelper::narrowScalarShift(MachineInstr &MI, unsigned TypeIdx, 3374 LLT RequestedTy) { 3375 if (TypeIdx == 1) { 3376 Observer.changingInstr(MI); 3377 narrowScalarSrc(MI, RequestedTy, 2); 3378 Observer.changedInstr(MI); 3379 return Legalized; 3380 } 3381 3382 Register DstReg = MI.getOperand(0).getReg(); 3383 LLT DstTy = MRI.getType(DstReg); 3384 if (DstTy.isVector()) 3385 return UnableToLegalize; 3386 3387 Register Amt = MI.getOperand(2).getReg(); 3388 LLT ShiftAmtTy = MRI.getType(Amt); 3389 const unsigned DstEltSize = DstTy.getScalarSizeInBits(); 3390 if (DstEltSize % 2 != 0) 3391 return UnableToLegalize; 3392 3393 // Ignore the input type. We can only go to exactly half the size of the 3394 // input. If that isn't small enough, the resulting pieces will be further 3395 // legalized. 3396 const unsigned NewBitSize = DstEltSize / 2; 3397 const LLT HalfTy = LLT::scalar(NewBitSize); 3398 const LLT CondTy = LLT::scalar(1); 3399 3400 if (const MachineInstr *KShiftAmt = 3401 getOpcodeDef(TargetOpcode::G_CONSTANT, Amt, MRI)) { 3402 return narrowScalarShiftByConstant( 3403 MI, KShiftAmt->getOperand(1).getCImm()->getValue(), HalfTy, ShiftAmtTy); 3404 } 3405 3406 // TODO: Expand with known bits. 3407 3408 // Handle the fully general expansion by an unknown amount. 3409 auto NewBits = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize); 3410 3411 Register InL = MRI.createGenericVirtualRegister(HalfTy); 3412 Register InH = MRI.createGenericVirtualRegister(HalfTy); 3413 MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1)); 3414 3415 auto AmtExcess = MIRBuilder.buildSub(ShiftAmtTy, Amt, NewBits); 3416 auto AmtLack = MIRBuilder.buildSub(ShiftAmtTy, NewBits, Amt); 3417 3418 auto Zero = MIRBuilder.buildConstant(ShiftAmtTy, 0); 3419 auto IsShort = MIRBuilder.buildICmp(ICmpInst::ICMP_ULT, CondTy, Amt, NewBits); 3420 auto IsZero = MIRBuilder.buildICmp(ICmpInst::ICMP_EQ, CondTy, Amt, Zero); 3421 3422 Register ResultRegs[2]; 3423 switch (MI.getOpcode()) { 3424 case TargetOpcode::G_SHL: { 3425 // Short: ShAmt < NewBitSize 3426 auto LoS = MIRBuilder.buildShl(HalfTy, InL, Amt); 3427 3428 auto LoOr = MIRBuilder.buildLShr(HalfTy, InL, AmtLack); 3429 auto HiOr = MIRBuilder.buildShl(HalfTy, InH, Amt); 3430 auto HiS = MIRBuilder.buildOr(HalfTy, LoOr, HiOr); 3431 3432 // Long: ShAmt >= NewBitSize 3433 auto LoL = MIRBuilder.buildConstant(HalfTy, 0); // Lo part is zero. 3434 auto HiL = MIRBuilder.buildShl(HalfTy, InL, AmtExcess); // Hi from Lo part. 3435 3436 auto Lo = MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL); 3437 auto Hi = MIRBuilder.buildSelect( 3438 HalfTy, IsZero, InH, MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL)); 3439 3440 ResultRegs[0] = Lo.getReg(0); 3441 ResultRegs[1] = Hi.getReg(0); 3442 break; 3443 } 3444 case TargetOpcode::G_LSHR: 3445 case TargetOpcode::G_ASHR: { 3446 // Short: ShAmt < NewBitSize 3447 auto HiS = MIRBuilder.buildInstr(MI.getOpcode(), {HalfTy}, {InH, Amt}); 3448 3449 auto LoOr = MIRBuilder.buildLShr(HalfTy, InL, Amt); 3450 auto HiOr = MIRBuilder.buildShl(HalfTy, InH, AmtLack); 3451 auto LoS = MIRBuilder.buildOr(HalfTy, LoOr, HiOr); 3452 3453 // Long: ShAmt >= NewBitSize 3454 MachineInstrBuilder HiL; 3455 if (MI.getOpcode() == TargetOpcode::G_LSHR) { 3456 HiL = MIRBuilder.buildConstant(HalfTy, 0); // Hi part is zero. 3457 } else { 3458 auto ShiftAmt = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize - 1); 3459 HiL = MIRBuilder.buildAShr(HalfTy, InH, ShiftAmt); // Sign of Hi part. 3460 } 3461 auto LoL = MIRBuilder.buildInstr(MI.getOpcode(), {HalfTy}, 3462 {InH, AmtExcess}); // Lo from Hi part. 3463 3464 auto Lo = MIRBuilder.buildSelect( 3465 HalfTy, IsZero, InL, MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL)); 3466 3467 auto Hi = MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL); 3468 3469 ResultRegs[0] = Lo.getReg(0); 3470 ResultRegs[1] = Hi.getReg(0); 3471 break; 3472 } 3473 default: 3474 llvm_unreachable("not a shift"); 3475 } 3476 3477 MIRBuilder.buildMerge(DstReg, ResultRegs); 3478 MI.eraseFromParent(); 3479 return Legalized; 3480 } 3481 3482 LegalizerHelper::LegalizeResult 3483 LegalizerHelper::moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx, 3484 LLT MoreTy) { 3485 assert(TypeIdx == 0 && "Expecting only Idx 0"); 3486 3487 Observer.changingInstr(MI); 3488 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) { 3489 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB(); 3490 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator()); 3491 moreElementsVectorSrc(MI, MoreTy, I); 3492 } 3493 3494 MachineBasicBlock &MBB = *MI.getParent(); 3495 MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI()); 3496 moreElementsVectorDst(MI, MoreTy, 0); 3497 Observer.changedInstr(MI); 3498 return Legalized; 3499 } 3500 3501 LegalizerHelper::LegalizeResult 3502 LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx, 3503 LLT MoreTy) { 3504 MIRBuilder.setInstr(MI); 3505 unsigned Opc = MI.getOpcode(); 3506 switch (Opc) { 3507 case TargetOpcode::G_IMPLICIT_DEF: 3508 case TargetOpcode::G_LOAD: { 3509 if (TypeIdx != 0) 3510 return UnableToLegalize; 3511 Observer.changingInstr(MI); 3512 moreElementsVectorDst(MI, MoreTy, 0); 3513 Observer.changedInstr(MI); 3514 return Legalized; 3515 } 3516 case TargetOpcode::G_STORE: 3517 if (TypeIdx != 0) 3518 return UnableToLegalize; 3519 Observer.changingInstr(MI); 3520 moreElementsVectorSrc(MI, MoreTy, 0); 3521 Observer.changedInstr(MI); 3522 return Legalized; 3523 case TargetOpcode::G_AND: 3524 case TargetOpcode::G_OR: 3525 case TargetOpcode::G_XOR: 3526 case TargetOpcode::G_SMIN: 3527 case TargetOpcode::G_SMAX: 3528 case TargetOpcode::G_UMIN: 3529 case TargetOpcode::G_UMAX: 3530 case TargetOpcode::G_FMINNUM: 3531 case TargetOpcode::G_FMAXNUM: 3532 case TargetOpcode::G_FMINNUM_IEEE: 3533 case TargetOpcode::G_FMAXNUM_IEEE: 3534 case TargetOpcode::G_FMINIMUM: 3535 case TargetOpcode::G_FMAXIMUM: { 3536 Observer.changingInstr(MI); 3537 moreElementsVectorSrc(MI, MoreTy, 1); 3538 moreElementsVectorSrc(MI, MoreTy, 2); 3539 moreElementsVectorDst(MI, MoreTy, 0); 3540 Observer.changedInstr(MI); 3541 return Legalized; 3542 } 3543 case TargetOpcode::G_EXTRACT: 3544 if (TypeIdx != 1) 3545 return UnableToLegalize; 3546 Observer.changingInstr(MI); 3547 moreElementsVectorSrc(MI, MoreTy, 1); 3548 Observer.changedInstr(MI); 3549 return Legalized; 3550 case TargetOpcode::G_INSERT: 3551 if (TypeIdx != 0) 3552 return UnableToLegalize; 3553 Observer.changingInstr(MI); 3554 moreElementsVectorSrc(MI, MoreTy, 1); 3555 moreElementsVectorDst(MI, MoreTy, 0); 3556 Observer.changedInstr(MI); 3557 return Legalized; 3558 case TargetOpcode::G_SELECT: 3559 if (TypeIdx != 0) 3560 return UnableToLegalize; 3561 if (MRI.getType(MI.getOperand(1).getReg()).isVector()) 3562 return UnableToLegalize; 3563 3564 Observer.changingInstr(MI); 3565 moreElementsVectorSrc(MI, MoreTy, 2); 3566 moreElementsVectorSrc(MI, MoreTy, 3); 3567 moreElementsVectorDst(MI, MoreTy, 0); 3568 Observer.changedInstr(MI); 3569 return Legalized; 3570 case TargetOpcode::G_UNMERGE_VALUES: { 3571 if (TypeIdx != 1) 3572 return UnableToLegalize; 3573 3574 LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); 3575 int NumDst = MI.getNumOperands() - 1; 3576 moreElementsVectorSrc(MI, MoreTy, NumDst); 3577 3578 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES); 3579 for (int I = 0; I != NumDst; ++I) 3580 MIB.addDef(MI.getOperand(I).getReg()); 3581 3582 int NewNumDst = MoreTy.getSizeInBits() / DstTy.getSizeInBits(); 3583 for (int I = NumDst; I != NewNumDst; ++I) 3584 MIB.addDef(MRI.createGenericVirtualRegister(DstTy)); 3585 3586 MIB.addUse(MI.getOperand(NumDst).getReg()); 3587 MI.eraseFromParent(); 3588 return Legalized; 3589 } 3590 case TargetOpcode::G_PHI: 3591 return moreElementsVectorPhi(MI, TypeIdx, MoreTy); 3592 default: 3593 return UnableToLegalize; 3594 } 3595 } 3596 3597 void LegalizerHelper::multiplyRegisters(SmallVectorImpl<Register> &DstRegs, 3598 ArrayRef<Register> Src1Regs, 3599 ArrayRef<Register> Src2Regs, 3600 LLT NarrowTy) { 3601 MachineIRBuilder &B = MIRBuilder; 3602 unsigned SrcParts = Src1Regs.size(); 3603 unsigned DstParts = DstRegs.size(); 3604 3605 unsigned DstIdx = 0; // Low bits of the result. 3606 Register FactorSum = 3607 B.buildMul(NarrowTy, Src1Regs[DstIdx], Src2Regs[DstIdx]).getReg(0); 3608 DstRegs[DstIdx] = FactorSum; 3609 3610 unsigned CarrySumPrevDstIdx; 3611 SmallVector<Register, 4> Factors; 3612 3613 for (DstIdx = 1; DstIdx < DstParts; DstIdx++) { 3614 // Collect low parts of muls for DstIdx. 3615 for (unsigned i = DstIdx + 1 < SrcParts ? 0 : DstIdx - SrcParts + 1; 3616 i <= std::min(DstIdx, SrcParts - 1); ++i) { 3617 MachineInstrBuilder Mul = 3618 B.buildMul(NarrowTy, Src1Regs[DstIdx - i], Src2Regs[i]); 3619 Factors.push_back(Mul.getReg(0)); 3620 } 3621 // Collect high parts of muls from previous DstIdx. 3622 for (unsigned i = DstIdx < SrcParts ? 0 : DstIdx - SrcParts; 3623 i <= std::min(DstIdx - 1, SrcParts - 1); ++i) { 3624 MachineInstrBuilder Umulh = 3625 B.buildUMulH(NarrowTy, Src1Regs[DstIdx - 1 - i], Src2Regs[i]); 3626 Factors.push_back(Umulh.getReg(0)); 3627 } 3628 // Add CarrySum from additions calculated for previous DstIdx. 3629 if (DstIdx != 1) { 3630 Factors.push_back(CarrySumPrevDstIdx); 3631 } 3632 3633 Register CarrySum; 3634 // Add all factors and accumulate all carries into CarrySum. 3635 if (DstIdx != DstParts - 1) { 3636 MachineInstrBuilder Uaddo = 3637 B.buildUAddo(NarrowTy, LLT::scalar(1), Factors[0], Factors[1]); 3638 FactorSum = Uaddo.getReg(0); 3639 CarrySum = B.buildZExt(NarrowTy, Uaddo.getReg(1)).getReg(0); 3640 for (unsigned i = 2; i < Factors.size(); ++i) { 3641 MachineInstrBuilder Uaddo = 3642 B.buildUAddo(NarrowTy, LLT::scalar(1), FactorSum, Factors[i]); 3643 FactorSum = Uaddo.getReg(0); 3644 MachineInstrBuilder Carry = B.buildZExt(NarrowTy, Uaddo.getReg(1)); 3645 CarrySum = B.buildAdd(NarrowTy, CarrySum, Carry).getReg(0); 3646 } 3647 } else { 3648 // Since value for the next index is not calculated, neither is CarrySum. 3649 FactorSum = B.buildAdd(NarrowTy, Factors[0], Factors[1]).getReg(0); 3650 for (unsigned i = 2; i < Factors.size(); ++i) 3651 FactorSum = B.buildAdd(NarrowTy, FactorSum, Factors[i]).getReg(0); 3652 } 3653 3654 CarrySumPrevDstIdx = CarrySum; 3655 DstRegs[DstIdx] = FactorSum; 3656 Factors.clear(); 3657 } 3658 } 3659 3660 LegalizerHelper::LegalizeResult 3661 LegalizerHelper::narrowScalarMul(MachineInstr &MI, LLT NarrowTy) { 3662 Register DstReg = MI.getOperand(0).getReg(); 3663 Register Src1 = MI.getOperand(1).getReg(); 3664 Register Src2 = MI.getOperand(2).getReg(); 3665 3666 LLT Ty = MRI.getType(DstReg); 3667 if (Ty.isVector()) 3668 return UnableToLegalize; 3669 3670 unsigned SrcSize = MRI.getType(Src1).getSizeInBits(); 3671 unsigned DstSize = Ty.getSizeInBits(); 3672 unsigned NarrowSize = NarrowTy.getSizeInBits(); 3673 if (DstSize % NarrowSize != 0 || SrcSize % NarrowSize != 0) 3674 return UnableToLegalize; 3675 3676 unsigned NumDstParts = DstSize / NarrowSize; 3677 unsigned NumSrcParts = SrcSize / NarrowSize; 3678 bool IsMulHigh = MI.getOpcode() == TargetOpcode::G_UMULH; 3679 unsigned DstTmpParts = NumDstParts * (IsMulHigh ? 2 : 1); 3680 3681 SmallVector<Register, 2> Src1Parts, Src2Parts; 3682 SmallVector<Register, 2> DstTmpRegs(DstTmpParts); 3683 extractParts(Src1, NarrowTy, NumSrcParts, Src1Parts); 3684 extractParts(Src2, NarrowTy, NumSrcParts, Src2Parts); 3685 multiplyRegisters(DstTmpRegs, Src1Parts, Src2Parts, NarrowTy); 3686 3687 // Take only high half of registers if this is high mul. 3688 ArrayRef<Register> DstRegs( 3689 IsMulHigh ? &DstTmpRegs[DstTmpParts / 2] : &DstTmpRegs[0], NumDstParts); 3690 MIRBuilder.buildMerge(DstReg, DstRegs); 3691 MI.eraseFromParent(); 3692 return Legalized; 3693 } 3694 3695 LegalizerHelper::LegalizeResult 3696 LegalizerHelper::narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx, 3697 LLT NarrowTy) { 3698 if (TypeIdx != 1) 3699 return UnableToLegalize; 3700 3701 uint64_t NarrowSize = NarrowTy.getSizeInBits(); 3702 3703 int64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits(); 3704 // FIXME: add support for when SizeOp1 isn't an exact multiple of 3705 // NarrowSize. 3706 if (SizeOp1 % NarrowSize != 0) 3707 return UnableToLegalize; 3708 int NumParts = SizeOp1 / NarrowSize; 3709 3710 SmallVector<Register, 2> SrcRegs, DstRegs; 3711 SmallVector<uint64_t, 2> Indexes; 3712 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs); 3713 3714 Register OpReg = MI.getOperand(0).getReg(); 3715 uint64_t OpStart = MI.getOperand(2).getImm(); 3716 uint64_t OpSize = MRI.getType(OpReg).getSizeInBits(); 3717 for (int i = 0; i < NumParts; ++i) { 3718 unsigned SrcStart = i * NarrowSize; 3719 3720 if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) { 3721 // No part of the extract uses this subregister, ignore it. 3722 continue; 3723 } else if (SrcStart == OpStart && NarrowTy == MRI.getType(OpReg)) { 3724 // The entire subregister is extracted, forward the value. 3725 DstRegs.push_back(SrcRegs[i]); 3726 continue; 3727 } 3728 3729 // OpSegStart is where this destination segment would start in OpReg if it 3730 // extended infinitely in both directions. 3731 int64_t ExtractOffset; 3732 uint64_t SegSize; 3733 if (OpStart < SrcStart) { 3734 ExtractOffset = 0; 3735 SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart); 3736 } else { 3737 ExtractOffset = OpStart - SrcStart; 3738 SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize); 3739 } 3740 3741 Register SegReg = SrcRegs[i]; 3742 if (ExtractOffset != 0 || SegSize != NarrowSize) { 3743 // A genuine extract is needed. 3744 SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize)); 3745 MIRBuilder.buildExtract(SegReg, SrcRegs[i], ExtractOffset); 3746 } 3747 3748 DstRegs.push_back(SegReg); 3749 } 3750 3751 Register DstReg = MI.getOperand(0).getReg(); 3752 if(MRI.getType(DstReg).isVector()) 3753 MIRBuilder.buildBuildVector(DstReg, DstRegs); 3754 else 3755 MIRBuilder.buildMerge(DstReg, DstRegs); 3756 MI.eraseFromParent(); 3757 return Legalized; 3758 } 3759 3760 LegalizerHelper::LegalizeResult 3761 LegalizerHelper::narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx, 3762 LLT NarrowTy) { 3763 // FIXME: Don't know how to handle secondary types yet. 3764 if (TypeIdx != 0) 3765 return UnableToLegalize; 3766 3767 uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 3768 uint64_t NarrowSize = NarrowTy.getSizeInBits(); 3769 3770 // FIXME: add support for when SizeOp0 isn't an exact multiple of 3771 // NarrowSize. 3772 if (SizeOp0 % NarrowSize != 0) 3773 return UnableToLegalize; 3774 3775 int NumParts = SizeOp0 / NarrowSize; 3776 3777 SmallVector<Register, 2> SrcRegs, DstRegs; 3778 SmallVector<uint64_t, 2> Indexes; 3779 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs); 3780 3781 Register OpReg = MI.getOperand(2).getReg(); 3782 uint64_t OpStart = MI.getOperand(3).getImm(); 3783 uint64_t OpSize = MRI.getType(OpReg).getSizeInBits(); 3784 for (int i = 0; i < NumParts; ++i) { 3785 unsigned DstStart = i * NarrowSize; 3786 3787 if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) { 3788 // No part of the insert affects this subregister, forward the original. 3789 DstRegs.push_back(SrcRegs[i]); 3790 continue; 3791 } else if (DstStart == OpStart && NarrowTy == MRI.getType(OpReg)) { 3792 // The entire subregister is defined by this insert, forward the new 3793 // value. 3794 DstRegs.push_back(OpReg); 3795 continue; 3796 } 3797 3798 // OpSegStart is where this destination segment would start in OpReg if it 3799 // extended infinitely in both directions. 3800 int64_t ExtractOffset, InsertOffset; 3801 uint64_t SegSize; 3802 if (OpStart < DstStart) { 3803 InsertOffset = 0; 3804 ExtractOffset = DstStart - OpStart; 3805 SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart); 3806 } else { 3807 InsertOffset = OpStart - DstStart; 3808 ExtractOffset = 0; 3809 SegSize = 3810 std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart); 3811 } 3812 3813 Register SegReg = OpReg; 3814 if (ExtractOffset != 0 || SegSize != OpSize) { 3815 // A genuine extract is needed. 3816 SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize)); 3817 MIRBuilder.buildExtract(SegReg, OpReg, ExtractOffset); 3818 } 3819 3820 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy); 3821 MIRBuilder.buildInsert(DstReg, SrcRegs[i], SegReg, InsertOffset); 3822 DstRegs.push_back(DstReg); 3823 } 3824 3825 assert(DstRegs.size() == (unsigned)NumParts && "not all parts covered"); 3826 Register DstReg = MI.getOperand(0).getReg(); 3827 if(MRI.getType(DstReg).isVector()) 3828 MIRBuilder.buildBuildVector(DstReg, DstRegs); 3829 else 3830 MIRBuilder.buildMerge(DstReg, DstRegs); 3831 MI.eraseFromParent(); 3832 return Legalized; 3833 } 3834 3835 LegalizerHelper::LegalizeResult 3836 LegalizerHelper::narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx, 3837 LLT NarrowTy) { 3838 Register DstReg = MI.getOperand(0).getReg(); 3839 LLT DstTy = MRI.getType(DstReg); 3840 3841 assert(MI.getNumOperands() == 3 && TypeIdx == 0); 3842 3843 SmallVector<Register, 4> DstRegs, DstLeftoverRegs; 3844 SmallVector<Register, 4> Src0Regs, Src0LeftoverRegs; 3845 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs; 3846 LLT LeftoverTy; 3847 if (!extractParts(MI.getOperand(1).getReg(), DstTy, NarrowTy, LeftoverTy, 3848 Src0Regs, Src0LeftoverRegs)) 3849 return UnableToLegalize; 3850 3851 LLT Unused; 3852 if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, Unused, 3853 Src1Regs, Src1LeftoverRegs)) 3854 llvm_unreachable("inconsistent extractParts result"); 3855 3856 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) { 3857 auto Inst = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy}, 3858 {Src0Regs[I], Src1Regs[I]}); 3859 DstRegs.push_back(Inst.getReg(0)); 3860 } 3861 3862 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) { 3863 auto Inst = MIRBuilder.buildInstr( 3864 MI.getOpcode(), 3865 {LeftoverTy}, {Src0LeftoverRegs[I], Src1LeftoverRegs[I]}); 3866 DstLeftoverRegs.push_back(Inst.getReg(0)); 3867 } 3868 3869 insertParts(DstReg, DstTy, NarrowTy, DstRegs, 3870 LeftoverTy, DstLeftoverRegs); 3871 3872 MI.eraseFromParent(); 3873 return Legalized; 3874 } 3875 3876 LegalizerHelper::LegalizeResult 3877 LegalizerHelper::narrowScalarExt(MachineInstr &MI, unsigned TypeIdx, 3878 LLT NarrowTy) { 3879 if (TypeIdx != 0) 3880 return UnableToLegalize; 3881 3882 Register DstReg = MI.getOperand(0).getReg(); 3883 Register SrcReg = MI.getOperand(1).getReg(); 3884 3885 LLT DstTy = MRI.getType(DstReg); 3886 if (DstTy.isVector()) 3887 return UnableToLegalize; 3888 3889 SmallVector<Register, 8> Parts; 3890 LLT GCDTy = extractGCDType(Parts, DstTy, NarrowTy, SrcReg); 3891 LLT LCMTy = buildLCMMergePieces(DstTy, NarrowTy, GCDTy, Parts, MI.getOpcode()); 3892 buildWidenedRemergeToDst(DstReg, LCMTy, Parts); 3893 3894 MI.eraseFromParent(); 3895 return Legalized; 3896 } 3897 3898 LegalizerHelper::LegalizeResult 3899 LegalizerHelper::narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx, 3900 LLT NarrowTy) { 3901 if (TypeIdx != 0) 3902 return UnableToLegalize; 3903 3904 Register CondReg = MI.getOperand(1).getReg(); 3905 LLT CondTy = MRI.getType(CondReg); 3906 if (CondTy.isVector()) // TODO: Handle vselect 3907 return UnableToLegalize; 3908 3909 Register DstReg = MI.getOperand(0).getReg(); 3910 LLT DstTy = MRI.getType(DstReg); 3911 3912 SmallVector<Register, 4> DstRegs, DstLeftoverRegs; 3913 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs; 3914 SmallVector<Register, 4> Src2Regs, Src2LeftoverRegs; 3915 LLT LeftoverTy; 3916 if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, LeftoverTy, 3917 Src1Regs, Src1LeftoverRegs)) 3918 return UnableToLegalize; 3919 3920 LLT Unused; 3921 if (!extractParts(MI.getOperand(3).getReg(), DstTy, NarrowTy, Unused, 3922 Src2Regs, Src2LeftoverRegs)) 3923 llvm_unreachable("inconsistent extractParts result"); 3924 3925 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) { 3926 auto Select = MIRBuilder.buildSelect(NarrowTy, 3927 CondReg, Src1Regs[I], Src2Regs[I]); 3928 DstRegs.push_back(Select.getReg(0)); 3929 } 3930 3931 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) { 3932 auto Select = MIRBuilder.buildSelect( 3933 LeftoverTy, CondReg, Src1LeftoverRegs[I], Src2LeftoverRegs[I]); 3934 DstLeftoverRegs.push_back(Select.getReg(0)); 3935 } 3936 3937 insertParts(DstReg, DstTy, NarrowTy, DstRegs, 3938 LeftoverTy, DstLeftoverRegs); 3939 3940 MI.eraseFromParent(); 3941 return Legalized; 3942 } 3943 3944 LegalizerHelper::LegalizeResult 3945 LegalizerHelper::narrowScalarCTLZ(MachineInstr &MI, unsigned TypeIdx, 3946 LLT NarrowTy) { 3947 if (TypeIdx != 1) 3948 return UnableToLegalize; 3949 3950 Register DstReg = MI.getOperand(0).getReg(); 3951 Register SrcReg = MI.getOperand(1).getReg(); 3952 LLT DstTy = MRI.getType(DstReg); 3953 LLT SrcTy = MRI.getType(SrcReg); 3954 unsigned NarrowSize = NarrowTy.getSizeInBits(); 3955 3956 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) { 3957 const bool IsUndef = MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF; 3958 3959 MachineIRBuilder &B = MIRBuilder; 3960 auto UnmergeSrc = B.buildUnmerge(NarrowTy, SrcReg); 3961 // ctlz(Hi:Lo) -> Hi == 0 ? (NarrowSize + ctlz(Lo)) : ctlz(Hi) 3962 auto C_0 = B.buildConstant(NarrowTy, 0); 3963 auto HiIsZero = B.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1), 3964 UnmergeSrc.getReg(1), C_0); 3965 auto LoCTLZ = IsUndef ? 3966 B.buildCTLZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(0)) : 3967 B.buildCTLZ(DstTy, UnmergeSrc.getReg(0)); 3968 auto C_NarrowSize = B.buildConstant(DstTy, NarrowSize); 3969 auto HiIsZeroCTLZ = B.buildAdd(DstTy, LoCTLZ, C_NarrowSize); 3970 auto HiCTLZ = B.buildCTLZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(1)); 3971 B.buildSelect(DstReg, HiIsZero, HiIsZeroCTLZ, HiCTLZ); 3972 3973 MI.eraseFromParent(); 3974 return Legalized; 3975 } 3976 3977 return UnableToLegalize; 3978 } 3979 3980 LegalizerHelper::LegalizeResult 3981 LegalizerHelper::narrowScalarCTTZ(MachineInstr &MI, unsigned TypeIdx, 3982 LLT NarrowTy) { 3983 if (TypeIdx != 1) 3984 return UnableToLegalize; 3985 3986 Register DstReg = MI.getOperand(0).getReg(); 3987 Register SrcReg = MI.getOperand(1).getReg(); 3988 LLT DstTy = MRI.getType(DstReg); 3989 LLT SrcTy = MRI.getType(SrcReg); 3990 unsigned NarrowSize = NarrowTy.getSizeInBits(); 3991 3992 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) { 3993 const bool IsUndef = MI.getOpcode() == TargetOpcode::G_CTTZ_ZERO_UNDEF; 3994 3995 MachineIRBuilder &B = MIRBuilder; 3996 auto UnmergeSrc = B.buildUnmerge(NarrowTy, SrcReg); 3997 // cttz(Hi:Lo) -> Lo == 0 ? (cttz(Hi) + NarrowSize) : cttz(Lo) 3998 auto C_0 = B.buildConstant(NarrowTy, 0); 3999 auto LoIsZero = B.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1), 4000 UnmergeSrc.getReg(0), C_0); 4001 auto HiCTTZ = IsUndef ? 4002 B.buildCTTZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(1)) : 4003 B.buildCTTZ(DstTy, UnmergeSrc.getReg(1)); 4004 auto C_NarrowSize = B.buildConstant(DstTy, NarrowSize); 4005 auto LoIsZeroCTTZ = B.buildAdd(DstTy, HiCTTZ, C_NarrowSize); 4006 auto LoCTTZ = B.buildCTTZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(0)); 4007 B.buildSelect(DstReg, LoIsZero, LoIsZeroCTTZ, LoCTTZ); 4008 4009 MI.eraseFromParent(); 4010 return Legalized; 4011 } 4012 4013 return UnableToLegalize; 4014 } 4015 4016 LegalizerHelper::LegalizeResult 4017 LegalizerHelper::narrowScalarCTPOP(MachineInstr &MI, unsigned TypeIdx, 4018 LLT NarrowTy) { 4019 if (TypeIdx != 1) 4020 return UnableToLegalize; 4021 4022 Register DstReg = MI.getOperand(0).getReg(); 4023 LLT DstTy = MRI.getType(DstReg); 4024 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg()); 4025 unsigned NarrowSize = NarrowTy.getSizeInBits(); 4026 4027 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) { 4028 auto UnmergeSrc = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1)); 4029 4030 auto LoCTPOP = MIRBuilder.buildCTPOP(DstTy, UnmergeSrc.getReg(0)); 4031 auto HiCTPOP = MIRBuilder.buildCTPOP(DstTy, UnmergeSrc.getReg(1)); 4032 MIRBuilder.buildAdd(DstReg, HiCTPOP, LoCTPOP); 4033 4034 MI.eraseFromParent(); 4035 return Legalized; 4036 } 4037 4038 return UnableToLegalize; 4039 } 4040 4041 LegalizerHelper::LegalizeResult 4042 LegalizerHelper::lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { 4043 unsigned Opc = MI.getOpcode(); 4044 auto &TII = *MI.getMF()->getSubtarget().getInstrInfo(); 4045 auto isSupported = [this](const LegalityQuery &Q) { 4046 auto QAction = LI.getAction(Q).Action; 4047 return QAction == Legal || QAction == Libcall || QAction == Custom; 4048 }; 4049 switch (Opc) { 4050 default: 4051 return UnableToLegalize; 4052 case TargetOpcode::G_CTLZ_ZERO_UNDEF: { 4053 // This trivially expands to CTLZ. 4054 Observer.changingInstr(MI); 4055 MI.setDesc(TII.get(TargetOpcode::G_CTLZ)); 4056 Observer.changedInstr(MI); 4057 return Legalized; 4058 } 4059 case TargetOpcode::G_CTLZ: { 4060 Register DstReg = MI.getOperand(0).getReg(); 4061 Register SrcReg = MI.getOperand(1).getReg(); 4062 LLT DstTy = MRI.getType(DstReg); 4063 LLT SrcTy = MRI.getType(SrcReg); 4064 unsigned Len = SrcTy.getSizeInBits(); 4065 4066 if (isSupported({TargetOpcode::G_CTLZ_ZERO_UNDEF, {DstTy, SrcTy}})) { 4067 // If CTLZ_ZERO_UNDEF is supported, emit that and a select for zero. 4068 auto CtlzZU = MIRBuilder.buildCTLZ_ZERO_UNDEF(DstTy, SrcReg); 4069 auto ZeroSrc = MIRBuilder.buildConstant(SrcTy, 0); 4070 auto ICmp = MIRBuilder.buildICmp( 4071 CmpInst::ICMP_EQ, SrcTy.changeElementSize(1), SrcReg, ZeroSrc); 4072 auto LenConst = MIRBuilder.buildConstant(DstTy, Len); 4073 MIRBuilder.buildSelect(DstReg, ICmp, LenConst, CtlzZU); 4074 MI.eraseFromParent(); 4075 return Legalized; 4076 } 4077 // for now, we do this: 4078 // NewLen = NextPowerOf2(Len); 4079 // x = x | (x >> 1); 4080 // x = x | (x >> 2); 4081 // ... 4082 // x = x | (x >>16); 4083 // x = x | (x >>32); // for 64-bit input 4084 // Upto NewLen/2 4085 // return Len - popcount(x); 4086 // 4087 // Ref: "Hacker's Delight" by Henry Warren 4088 Register Op = SrcReg; 4089 unsigned NewLen = PowerOf2Ceil(Len); 4090 for (unsigned i = 0; (1U << i) <= (NewLen / 2); ++i) { 4091 auto MIBShiftAmt = MIRBuilder.buildConstant(SrcTy, 1ULL << i); 4092 auto MIBOp = MIRBuilder.buildOr( 4093 SrcTy, Op, MIRBuilder.buildLShr(SrcTy, Op, MIBShiftAmt)); 4094 Op = MIBOp.getReg(0); 4095 } 4096 auto MIBPop = MIRBuilder.buildCTPOP(DstTy, Op); 4097 MIRBuilder.buildSub(MI.getOperand(0), MIRBuilder.buildConstant(DstTy, Len), 4098 MIBPop); 4099 MI.eraseFromParent(); 4100 return Legalized; 4101 } 4102 case TargetOpcode::G_CTTZ_ZERO_UNDEF: { 4103 // This trivially expands to CTTZ. 4104 Observer.changingInstr(MI); 4105 MI.setDesc(TII.get(TargetOpcode::G_CTTZ)); 4106 Observer.changedInstr(MI); 4107 return Legalized; 4108 } 4109 case TargetOpcode::G_CTTZ: { 4110 Register DstReg = MI.getOperand(0).getReg(); 4111 Register SrcReg = MI.getOperand(1).getReg(); 4112 LLT DstTy = MRI.getType(DstReg); 4113 LLT SrcTy = MRI.getType(SrcReg); 4114 4115 unsigned Len = SrcTy.getSizeInBits(); 4116 if (isSupported({TargetOpcode::G_CTTZ_ZERO_UNDEF, {DstTy, SrcTy}})) { 4117 // If CTTZ_ZERO_UNDEF is legal or custom, emit that and a select with 4118 // zero. 4119 auto CttzZU = MIRBuilder.buildCTTZ_ZERO_UNDEF(DstTy, SrcReg); 4120 auto Zero = MIRBuilder.buildConstant(SrcTy, 0); 4121 auto ICmp = MIRBuilder.buildICmp( 4122 CmpInst::ICMP_EQ, DstTy.changeElementSize(1), SrcReg, Zero); 4123 auto LenConst = MIRBuilder.buildConstant(DstTy, Len); 4124 MIRBuilder.buildSelect(DstReg, ICmp, LenConst, CttzZU); 4125 MI.eraseFromParent(); 4126 return Legalized; 4127 } 4128 // for now, we use: { return popcount(~x & (x - 1)); } 4129 // unless the target has ctlz but not ctpop, in which case we use: 4130 // { return 32 - nlz(~x & (x-1)); } 4131 // Ref: "Hacker's Delight" by Henry Warren 4132 auto MIBCstNeg1 = MIRBuilder.buildConstant(Ty, -1); 4133 auto MIBNot = MIRBuilder.buildXor(Ty, SrcReg, MIBCstNeg1); 4134 auto MIBTmp = MIRBuilder.buildAnd( 4135 Ty, MIBNot, MIRBuilder.buildAdd(Ty, SrcReg, MIBCstNeg1)); 4136 if (!isSupported({TargetOpcode::G_CTPOP, {Ty, Ty}}) && 4137 isSupported({TargetOpcode::G_CTLZ, {Ty, Ty}})) { 4138 auto MIBCstLen = MIRBuilder.buildConstant(Ty, Len); 4139 MIRBuilder.buildSub(MI.getOperand(0), MIBCstLen, 4140 MIRBuilder.buildCTLZ(Ty, MIBTmp)); 4141 MI.eraseFromParent(); 4142 return Legalized; 4143 } 4144 MI.setDesc(TII.get(TargetOpcode::G_CTPOP)); 4145 MI.getOperand(1).setReg(MIBTmp.getReg(0)); 4146 return Legalized; 4147 } 4148 case TargetOpcode::G_CTPOP: { 4149 unsigned Size = Ty.getSizeInBits(); 4150 MachineIRBuilder &B = MIRBuilder; 4151 4152 // Count set bits in blocks of 2 bits. Default approach would be 4153 // B2Count = { val & 0x55555555 } + { (val >> 1) & 0x55555555 } 4154 // We use following formula instead: 4155 // B2Count = val - { (val >> 1) & 0x55555555 } 4156 // since it gives same result in blocks of 2 with one instruction less. 4157 auto C_1 = B.buildConstant(Ty, 1); 4158 auto B2Set1LoTo1Hi = B.buildLShr(Ty, MI.getOperand(1).getReg(), C_1); 4159 APInt B2Mask1HiTo0 = APInt::getSplat(Size, APInt(8, 0x55)); 4160 auto C_B2Mask1HiTo0 = B.buildConstant(Ty, B2Mask1HiTo0); 4161 auto B2Count1Hi = B.buildAnd(Ty, B2Set1LoTo1Hi, C_B2Mask1HiTo0); 4162 auto B2Count = B.buildSub(Ty, MI.getOperand(1).getReg(), B2Count1Hi); 4163 4164 // In order to get count in blocks of 4 add values from adjacent block of 2. 4165 // B4Count = { B2Count & 0x33333333 } + { (B2Count >> 2) & 0x33333333 } 4166 auto C_2 = B.buildConstant(Ty, 2); 4167 auto B4Set2LoTo2Hi = B.buildLShr(Ty, B2Count, C_2); 4168 APInt B4Mask2HiTo0 = APInt::getSplat(Size, APInt(8, 0x33)); 4169 auto C_B4Mask2HiTo0 = B.buildConstant(Ty, B4Mask2HiTo0); 4170 auto B4HiB2Count = B.buildAnd(Ty, B4Set2LoTo2Hi, C_B4Mask2HiTo0); 4171 auto B4LoB2Count = B.buildAnd(Ty, B2Count, C_B4Mask2HiTo0); 4172 auto B4Count = B.buildAdd(Ty, B4HiB2Count, B4LoB2Count); 4173 4174 // For count in blocks of 8 bits we don't have to mask high 4 bits before 4175 // addition since count value sits in range {0,...,8} and 4 bits are enough 4176 // to hold such binary values. After addition high 4 bits still hold count 4177 // of set bits in high 4 bit block, set them to zero and get 8 bit result. 4178 // B8Count = { B4Count + (B4Count >> 4) } & 0x0F0F0F0F 4179 auto C_4 = B.buildConstant(Ty, 4); 4180 auto B8HiB4Count = B.buildLShr(Ty, B4Count, C_4); 4181 auto B8CountDirty4Hi = B.buildAdd(Ty, B8HiB4Count, B4Count); 4182 APInt B8Mask4HiTo0 = APInt::getSplat(Size, APInt(8, 0x0F)); 4183 auto C_B8Mask4HiTo0 = B.buildConstant(Ty, B8Mask4HiTo0); 4184 auto B8Count = B.buildAnd(Ty, B8CountDirty4Hi, C_B8Mask4HiTo0); 4185 4186 assert(Size<=128 && "Scalar size is too large for CTPOP lower algorithm"); 4187 // 8 bits can hold CTPOP result of 128 bit int or smaller. Mul with this 4188 // bitmask will set 8 msb in ResTmp to sum of all B8Counts in 8 bit blocks. 4189 auto MulMask = B.buildConstant(Ty, APInt::getSplat(Size, APInt(8, 0x01))); 4190 auto ResTmp = B.buildMul(Ty, B8Count, MulMask); 4191 4192 // Shift count result from 8 high bits to low bits. 4193 auto C_SizeM8 = B.buildConstant(Ty, Size - 8); 4194 B.buildLShr(MI.getOperand(0).getReg(), ResTmp, C_SizeM8); 4195 4196 MI.eraseFromParent(); 4197 return Legalized; 4198 } 4199 } 4200 } 4201 4202 // Expand s32 = G_UITOFP s64 using bit operations to an IEEE float 4203 // representation. 4204 LegalizerHelper::LegalizeResult 4205 LegalizerHelper::lowerU64ToF32BitOps(MachineInstr &MI) { 4206 Register Dst = MI.getOperand(0).getReg(); 4207 Register Src = MI.getOperand(1).getReg(); 4208 const LLT S64 = LLT::scalar(64); 4209 const LLT S32 = LLT::scalar(32); 4210 const LLT S1 = LLT::scalar(1); 4211 4212 assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S32); 4213 4214 // unsigned cul2f(ulong u) { 4215 // uint lz = clz(u); 4216 // uint e = (u != 0) ? 127U + 63U - lz : 0; 4217 // u = (u << lz) & 0x7fffffffffffffffUL; 4218 // ulong t = u & 0xffffffffffUL; 4219 // uint v = (e << 23) | (uint)(u >> 40); 4220 // uint r = t > 0x8000000000UL ? 1U : (t == 0x8000000000UL ? v & 1U : 0U); 4221 // return as_float(v + r); 4222 // } 4223 4224 auto Zero32 = MIRBuilder.buildConstant(S32, 0); 4225 auto Zero64 = MIRBuilder.buildConstant(S64, 0); 4226 4227 auto LZ = MIRBuilder.buildCTLZ_ZERO_UNDEF(S32, Src); 4228 4229 auto K = MIRBuilder.buildConstant(S32, 127U + 63U); 4230 auto Sub = MIRBuilder.buildSub(S32, K, LZ); 4231 4232 auto NotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, Src, Zero64); 4233 auto E = MIRBuilder.buildSelect(S32, NotZero, Sub, Zero32); 4234 4235 auto Mask0 = MIRBuilder.buildConstant(S64, (-1ULL) >> 1); 4236 auto ShlLZ = MIRBuilder.buildShl(S64, Src, LZ); 4237 4238 auto U = MIRBuilder.buildAnd(S64, ShlLZ, Mask0); 4239 4240 auto Mask1 = MIRBuilder.buildConstant(S64, 0xffffffffffULL); 4241 auto T = MIRBuilder.buildAnd(S64, U, Mask1); 4242 4243 auto UShl = MIRBuilder.buildLShr(S64, U, MIRBuilder.buildConstant(S64, 40)); 4244 auto ShlE = MIRBuilder.buildShl(S32, E, MIRBuilder.buildConstant(S32, 23)); 4245 auto V = MIRBuilder.buildOr(S32, ShlE, MIRBuilder.buildTrunc(S32, UShl)); 4246 4247 auto C = MIRBuilder.buildConstant(S64, 0x8000000000ULL); 4248 auto RCmp = MIRBuilder.buildICmp(CmpInst::ICMP_UGT, S1, T, C); 4249 auto TCmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, T, C); 4250 auto One = MIRBuilder.buildConstant(S32, 1); 4251 4252 auto VTrunc1 = MIRBuilder.buildAnd(S32, V, One); 4253 auto Select0 = MIRBuilder.buildSelect(S32, TCmp, VTrunc1, Zero32); 4254 auto R = MIRBuilder.buildSelect(S32, RCmp, One, Select0); 4255 MIRBuilder.buildAdd(Dst, V, R); 4256 4257 return Legalized; 4258 } 4259 4260 LegalizerHelper::LegalizeResult 4261 LegalizerHelper::lowerUITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { 4262 Register Dst = MI.getOperand(0).getReg(); 4263 Register Src = MI.getOperand(1).getReg(); 4264 LLT DstTy = MRI.getType(Dst); 4265 LLT SrcTy = MRI.getType(Src); 4266 4267 if (SrcTy == LLT::scalar(1)) { 4268 auto True = MIRBuilder.buildFConstant(DstTy, 1.0); 4269 auto False = MIRBuilder.buildFConstant(DstTy, 0.0); 4270 MIRBuilder.buildSelect(Dst, Src, True, False); 4271 MI.eraseFromParent(); 4272 return Legalized; 4273 } 4274 4275 if (SrcTy != LLT::scalar(64)) 4276 return UnableToLegalize; 4277 4278 if (DstTy == LLT::scalar(32)) { 4279 // TODO: SelectionDAG has several alternative expansions to port which may 4280 // be more reasonble depending on the available instructions. If a target 4281 // has sitofp, does not have CTLZ, or can efficiently use f64 as an 4282 // intermediate type, this is probably worse. 4283 return lowerU64ToF32BitOps(MI); 4284 } 4285 4286 return UnableToLegalize; 4287 } 4288 4289 LegalizerHelper::LegalizeResult 4290 LegalizerHelper::lowerSITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { 4291 Register Dst = MI.getOperand(0).getReg(); 4292 Register Src = MI.getOperand(1).getReg(); 4293 LLT DstTy = MRI.getType(Dst); 4294 LLT SrcTy = MRI.getType(Src); 4295 4296 const LLT S64 = LLT::scalar(64); 4297 const LLT S32 = LLT::scalar(32); 4298 const LLT S1 = LLT::scalar(1); 4299 4300 if (SrcTy == S1) { 4301 auto True = MIRBuilder.buildFConstant(DstTy, -1.0); 4302 auto False = MIRBuilder.buildFConstant(DstTy, 0.0); 4303 MIRBuilder.buildSelect(Dst, Src, True, False); 4304 MI.eraseFromParent(); 4305 return Legalized; 4306 } 4307 4308 if (SrcTy != S64) 4309 return UnableToLegalize; 4310 4311 if (DstTy == S32) { 4312 // signed cl2f(long l) { 4313 // long s = l >> 63; 4314 // float r = cul2f((l + s) ^ s); 4315 // return s ? -r : r; 4316 // } 4317 Register L = Src; 4318 auto SignBit = MIRBuilder.buildConstant(S64, 63); 4319 auto S = MIRBuilder.buildAShr(S64, L, SignBit); 4320 4321 auto LPlusS = MIRBuilder.buildAdd(S64, L, S); 4322 auto Xor = MIRBuilder.buildXor(S64, LPlusS, S); 4323 auto R = MIRBuilder.buildUITOFP(S32, Xor); 4324 4325 auto RNeg = MIRBuilder.buildFNeg(S32, R); 4326 auto SignNotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, S, 4327 MIRBuilder.buildConstant(S64, 0)); 4328 MIRBuilder.buildSelect(Dst, SignNotZero, RNeg, R); 4329 return Legalized; 4330 } 4331 4332 return UnableToLegalize; 4333 } 4334 4335 LegalizerHelper::LegalizeResult 4336 LegalizerHelper::lowerFPTOUI(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { 4337 Register Dst = MI.getOperand(0).getReg(); 4338 Register Src = MI.getOperand(1).getReg(); 4339 LLT DstTy = MRI.getType(Dst); 4340 LLT SrcTy = MRI.getType(Src); 4341 const LLT S64 = LLT::scalar(64); 4342 const LLT S32 = LLT::scalar(32); 4343 4344 if (SrcTy != S64 && SrcTy != S32) 4345 return UnableToLegalize; 4346 if (DstTy != S32 && DstTy != S64) 4347 return UnableToLegalize; 4348 4349 // FPTOSI gives same result as FPTOUI for positive signed integers. 4350 // FPTOUI needs to deal with fp values that convert to unsigned integers 4351 // greater or equal to 2^31 for float or 2^63 for double. For brevity 2^Exp. 4352 4353 APInt TwoPExpInt = APInt::getSignMask(DstTy.getSizeInBits()); 4354 APFloat TwoPExpFP(SrcTy.getSizeInBits() == 32 ? APFloat::IEEEsingle() 4355 : APFloat::IEEEdouble(), 4356 APInt::getNullValue(SrcTy.getSizeInBits())); 4357 TwoPExpFP.convertFromAPInt(TwoPExpInt, false, APFloat::rmNearestTiesToEven); 4358 4359 MachineInstrBuilder FPTOSI = MIRBuilder.buildFPTOSI(DstTy, Src); 4360 4361 MachineInstrBuilder Threshold = MIRBuilder.buildFConstant(SrcTy, TwoPExpFP); 4362 // For fp Value greater or equal to Threshold(2^Exp), we use FPTOSI on 4363 // (Value - 2^Exp) and add 2^Exp by setting highest bit in result to 1. 4364 MachineInstrBuilder FSub = MIRBuilder.buildFSub(SrcTy, Src, Threshold); 4365 MachineInstrBuilder ResLowBits = MIRBuilder.buildFPTOSI(DstTy, FSub); 4366 MachineInstrBuilder ResHighBit = MIRBuilder.buildConstant(DstTy, TwoPExpInt); 4367 MachineInstrBuilder Res = MIRBuilder.buildXor(DstTy, ResLowBits, ResHighBit); 4368 4369 const LLT S1 = LLT::scalar(1); 4370 4371 MachineInstrBuilder FCMP = 4372 MIRBuilder.buildFCmp(CmpInst::FCMP_ULT, S1, Src, Threshold); 4373 MIRBuilder.buildSelect(Dst, FCMP, FPTOSI, Res); 4374 4375 MI.eraseFromParent(); 4376 return Legalized; 4377 } 4378 4379 LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPTOSI(MachineInstr &MI) { 4380 Register Dst = MI.getOperand(0).getReg(); 4381 Register Src = MI.getOperand(1).getReg(); 4382 LLT DstTy = MRI.getType(Dst); 4383 LLT SrcTy = MRI.getType(Src); 4384 const LLT S64 = LLT::scalar(64); 4385 const LLT S32 = LLT::scalar(32); 4386 4387 // FIXME: Only f32 to i64 conversions are supported. 4388 if (SrcTy.getScalarType() != S32 || DstTy.getScalarType() != S64) 4389 return UnableToLegalize; 4390 4391 // Expand f32 -> i64 conversion 4392 // This algorithm comes from compiler-rt's implementation of fixsfdi: 4393 // https://github.com/llvm/llvm-project/blob/master/compiler-rt/lib/builtins/fixsfdi.c 4394 4395 unsigned SrcEltBits = SrcTy.getScalarSizeInBits(); 4396 4397 auto ExponentMask = MIRBuilder.buildConstant(SrcTy, 0x7F800000); 4398 auto ExponentLoBit = MIRBuilder.buildConstant(SrcTy, 23); 4399 4400 auto AndExpMask = MIRBuilder.buildAnd(SrcTy, Src, ExponentMask); 4401 auto ExponentBits = MIRBuilder.buildLShr(SrcTy, AndExpMask, ExponentLoBit); 4402 4403 auto SignMask = MIRBuilder.buildConstant(SrcTy, 4404 APInt::getSignMask(SrcEltBits)); 4405 auto AndSignMask = MIRBuilder.buildAnd(SrcTy, Src, SignMask); 4406 auto SignLowBit = MIRBuilder.buildConstant(SrcTy, SrcEltBits - 1); 4407 auto Sign = MIRBuilder.buildAShr(SrcTy, AndSignMask, SignLowBit); 4408 Sign = MIRBuilder.buildSExt(DstTy, Sign); 4409 4410 auto MantissaMask = MIRBuilder.buildConstant(SrcTy, 0x007FFFFF); 4411 auto AndMantissaMask = MIRBuilder.buildAnd(SrcTy, Src, MantissaMask); 4412 auto K = MIRBuilder.buildConstant(SrcTy, 0x00800000); 4413 4414 auto R = MIRBuilder.buildOr(SrcTy, AndMantissaMask, K); 4415 R = MIRBuilder.buildZExt(DstTy, R); 4416 4417 auto Bias = MIRBuilder.buildConstant(SrcTy, 127); 4418 auto Exponent = MIRBuilder.buildSub(SrcTy, ExponentBits, Bias); 4419 auto SubExponent = MIRBuilder.buildSub(SrcTy, Exponent, ExponentLoBit); 4420 auto ExponentSub = MIRBuilder.buildSub(SrcTy, ExponentLoBit, Exponent); 4421 4422 auto Shl = MIRBuilder.buildShl(DstTy, R, SubExponent); 4423 auto Srl = MIRBuilder.buildLShr(DstTy, R, ExponentSub); 4424 4425 const LLT S1 = LLT::scalar(1); 4426 auto CmpGt = MIRBuilder.buildICmp(CmpInst::ICMP_SGT, 4427 S1, Exponent, ExponentLoBit); 4428 4429 R = MIRBuilder.buildSelect(DstTy, CmpGt, Shl, Srl); 4430 4431 auto XorSign = MIRBuilder.buildXor(DstTy, R, Sign); 4432 auto Ret = MIRBuilder.buildSub(DstTy, XorSign, Sign); 4433 4434 auto ZeroSrcTy = MIRBuilder.buildConstant(SrcTy, 0); 4435 4436 auto ExponentLt0 = MIRBuilder.buildICmp(CmpInst::ICMP_SLT, 4437 S1, Exponent, ZeroSrcTy); 4438 4439 auto ZeroDstTy = MIRBuilder.buildConstant(DstTy, 0); 4440 MIRBuilder.buildSelect(Dst, ExponentLt0, ZeroDstTy, Ret); 4441 4442 MI.eraseFromParent(); 4443 return Legalized; 4444 } 4445 4446 // f64 -> f16 conversion using round-to-nearest-even rounding mode. 4447 LegalizerHelper::LegalizeResult 4448 LegalizerHelper::lowerFPTRUNC_F64_TO_F16(MachineInstr &MI) { 4449 Register Dst = MI.getOperand(0).getReg(); 4450 Register Src = MI.getOperand(1).getReg(); 4451 4452 if (MRI.getType(Src).isVector()) // TODO: Handle vectors directly. 4453 return UnableToLegalize; 4454 4455 const unsigned ExpMask = 0x7ff; 4456 const unsigned ExpBiasf64 = 1023; 4457 const unsigned ExpBiasf16 = 15; 4458 const LLT S32 = LLT::scalar(32); 4459 const LLT S1 = LLT::scalar(1); 4460 4461 auto Unmerge = MIRBuilder.buildUnmerge(S32, Src); 4462 Register U = Unmerge.getReg(0); 4463 Register UH = Unmerge.getReg(1); 4464 4465 auto E = MIRBuilder.buildLShr(S32, UH, MIRBuilder.buildConstant(S32, 20)); 4466 4467 // Subtract the fp64 exponent bias (1023) to get the real exponent and 4468 // add the f16 bias (15) to get the biased exponent for the f16 format. 4469 E = MIRBuilder.buildAdd( 4470 S32, E, MIRBuilder.buildConstant(S32, -ExpBiasf64 + ExpBiasf16)); 4471 E = MIRBuilder.buildAnd(S32, E, MIRBuilder.buildConstant(S32, ExpMask)); 4472 4473 auto M = MIRBuilder.buildLShr(S32, UH, MIRBuilder.buildConstant(S32, 8)); 4474 M = MIRBuilder.buildAnd(S32, M, MIRBuilder.buildConstant(S32, 0xffe)); 4475 4476 auto MaskedSig = MIRBuilder.buildAnd(S32, UH, 4477 MIRBuilder.buildConstant(S32, 0x1ff)); 4478 MaskedSig = MIRBuilder.buildOr(S32, MaskedSig, U); 4479 4480 auto Zero = MIRBuilder.buildConstant(S32, 0); 4481 auto SigCmpNE0 = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, MaskedSig, Zero); 4482 auto Lo40Set = MIRBuilder.buildZExt(S32, SigCmpNE0); 4483 M = MIRBuilder.buildOr(S32, M, Lo40Set); 4484 4485 // (M != 0 ? 0x0200 : 0) | 0x7c00; 4486 auto Bits0x200 = MIRBuilder.buildConstant(S32, 0x0200); 4487 auto CmpM_NE0 = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, M, Zero); 4488 auto SelectCC = MIRBuilder.buildSelect(S32, CmpM_NE0, Bits0x200, Zero); 4489 4490 auto Bits0x7c00 = MIRBuilder.buildConstant(S32, 0x7c00); 4491 auto I = MIRBuilder.buildOr(S32, SelectCC, Bits0x7c00); 4492 4493 // N = M | (E << 12); 4494 auto EShl12 = MIRBuilder.buildShl(S32, E, MIRBuilder.buildConstant(S32, 12)); 4495 auto N = MIRBuilder.buildOr(S32, M, EShl12); 4496 4497 // B = clamp(1-E, 0, 13); 4498 auto One = MIRBuilder.buildConstant(S32, 1); 4499 auto OneSubExp = MIRBuilder.buildSub(S32, One, E); 4500 auto B = MIRBuilder.buildSMax(S32, OneSubExp, Zero); 4501 B = MIRBuilder.buildSMin(S32, B, MIRBuilder.buildConstant(S32, 13)); 4502 4503 auto SigSetHigh = MIRBuilder.buildOr(S32, M, 4504 MIRBuilder.buildConstant(S32, 0x1000)); 4505 4506 auto D = MIRBuilder.buildLShr(S32, SigSetHigh, B); 4507 auto D0 = MIRBuilder.buildShl(S32, D, B); 4508 4509 auto D0_NE_SigSetHigh = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, 4510 D0, SigSetHigh); 4511 auto D1 = MIRBuilder.buildZExt(S32, D0_NE_SigSetHigh); 4512 D = MIRBuilder.buildOr(S32, D, D1); 4513 4514 auto CmpELtOne = MIRBuilder.buildICmp(CmpInst::ICMP_SLT, S1, E, One); 4515 auto V = MIRBuilder.buildSelect(S32, CmpELtOne, D, N); 4516 4517 auto VLow3 = MIRBuilder.buildAnd(S32, V, MIRBuilder.buildConstant(S32, 7)); 4518 V = MIRBuilder.buildLShr(S32, V, MIRBuilder.buildConstant(S32, 2)); 4519 4520 auto VLow3Eq3 = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, VLow3, 4521 MIRBuilder.buildConstant(S32, 3)); 4522 auto V0 = MIRBuilder.buildZExt(S32, VLow3Eq3); 4523 4524 auto VLow3Gt5 = MIRBuilder.buildICmp(CmpInst::ICMP_SGT, S1, VLow3, 4525 MIRBuilder.buildConstant(S32, 5)); 4526 auto V1 = MIRBuilder.buildZExt(S32, VLow3Gt5); 4527 4528 V1 = MIRBuilder.buildOr(S32, V0, V1); 4529 V = MIRBuilder.buildAdd(S32, V, V1); 4530 4531 auto CmpEGt30 = MIRBuilder.buildICmp(CmpInst::ICMP_SGT, S1, 4532 E, MIRBuilder.buildConstant(S32, 30)); 4533 V = MIRBuilder.buildSelect(S32, CmpEGt30, 4534 MIRBuilder.buildConstant(S32, 0x7c00), V); 4535 4536 auto CmpEGt1039 = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, 4537 E, MIRBuilder.buildConstant(S32, 1039)); 4538 V = MIRBuilder.buildSelect(S32, CmpEGt1039, I, V); 4539 4540 // Extract the sign bit. 4541 auto Sign = MIRBuilder.buildLShr(S32, UH, MIRBuilder.buildConstant(S32, 16)); 4542 Sign = MIRBuilder.buildAnd(S32, Sign, MIRBuilder.buildConstant(S32, 0x8000)); 4543 4544 // Insert the sign bit 4545 V = MIRBuilder.buildOr(S32, Sign, V); 4546 4547 MIRBuilder.buildTrunc(Dst, V); 4548 MI.eraseFromParent(); 4549 return Legalized; 4550 } 4551 4552 LegalizerHelper::LegalizeResult 4553 LegalizerHelper::lowerFPTRUNC(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { 4554 Register Dst = MI.getOperand(0).getReg(); 4555 Register Src = MI.getOperand(1).getReg(); 4556 4557 LLT DstTy = MRI.getType(Dst); 4558 LLT SrcTy = MRI.getType(Src); 4559 const LLT S64 = LLT::scalar(64); 4560 const LLT S16 = LLT::scalar(16); 4561 4562 if (DstTy.getScalarType() == S16 && SrcTy.getScalarType() == S64) 4563 return lowerFPTRUNC_F64_TO_F16(MI); 4564 4565 return UnableToLegalize; 4566 } 4567 4568 static CmpInst::Predicate minMaxToCompare(unsigned Opc) { 4569 switch (Opc) { 4570 case TargetOpcode::G_SMIN: 4571 return CmpInst::ICMP_SLT; 4572 case TargetOpcode::G_SMAX: 4573 return CmpInst::ICMP_SGT; 4574 case TargetOpcode::G_UMIN: 4575 return CmpInst::ICMP_ULT; 4576 case TargetOpcode::G_UMAX: 4577 return CmpInst::ICMP_UGT; 4578 default: 4579 llvm_unreachable("not in integer min/max"); 4580 } 4581 } 4582 4583 LegalizerHelper::LegalizeResult 4584 LegalizerHelper::lowerMinMax(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { 4585 Register Dst = MI.getOperand(0).getReg(); 4586 Register Src0 = MI.getOperand(1).getReg(); 4587 Register Src1 = MI.getOperand(2).getReg(); 4588 4589 const CmpInst::Predicate Pred = minMaxToCompare(MI.getOpcode()); 4590 LLT CmpType = MRI.getType(Dst).changeElementSize(1); 4591 4592 auto Cmp = MIRBuilder.buildICmp(Pred, CmpType, Src0, Src1); 4593 MIRBuilder.buildSelect(Dst, Cmp, Src0, Src1); 4594 4595 MI.eraseFromParent(); 4596 return Legalized; 4597 } 4598 4599 LegalizerHelper::LegalizeResult 4600 LegalizerHelper::lowerFCopySign(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { 4601 Register Dst = MI.getOperand(0).getReg(); 4602 Register Src0 = MI.getOperand(1).getReg(); 4603 Register Src1 = MI.getOperand(2).getReg(); 4604 4605 const LLT Src0Ty = MRI.getType(Src0); 4606 const LLT Src1Ty = MRI.getType(Src1); 4607 4608 const int Src0Size = Src0Ty.getScalarSizeInBits(); 4609 const int Src1Size = Src1Ty.getScalarSizeInBits(); 4610 4611 auto SignBitMask = MIRBuilder.buildConstant( 4612 Src0Ty, APInt::getSignMask(Src0Size)); 4613 4614 auto NotSignBitMask = MIRBuilder.buildConstant( 4615 Src0Ty, APInt::getLowBitsSet(Src0Size, Src0Size - 1)); 4616 4617 auto And0 = MIRBuilder.buildAnd(Src0Ty, Src0, NotSignBitMask); 4618 MachineInstr *Or; 4619 4620 if (Src0Ty == Src1Ty) { 4621 auto And1 = MIRBuilder.buildAnd(Src1Ty, Src0, SignBitMask); 4622 Or = MIRBuilder.buildOr(Dst, And0, And1); 4623 } else if (Src0Size > Src1Size) { 4624 auto ShiftAmt = MIRBuilder.buildConstant(Src0Ty, Src0Size - Src1Size); 4625 auto Zext = MIRBuilder.buildZExt(Src0Ty, Src1); 4626 auto Shift = MIRBuilder.buildShl(Src0Ty, Zext, ShiftAmt); 4627 auto And1 = MIRBuilder.buildAnd(Src0Ty, Shift, SignBitMask); 4628 Or = MIRBuilder.buildOr(Dst, And0, And1); 4629 } else { 4630 auto ShiftAmt = MIRBuilder.buildConstant(Src1Ty, Src1Size - Src0Size); 4631 auto Shift = MIRBuilder.buildLShr(Src1Ty, Src1, ShiftAmt); 4632 auto Trunc = MIRBuilder.buildTrunc(Src0Ty, Shift); 4633 auto And1 = MIRBuilder.buildAnd(Src0Ty, Trunc, SignBitMask); 4634 Or = MIRBuilder.buildOr(Dst, And0, And1); 4635 } 4636 4637 // Be careful about setting nsz/nnan/ninf on every instruction, since the 4638 // constants are a nan and -0.0, but the final result should preserve 4639 // everything. 4640 if (unsigned Flags = MI.getFlags()) 4641 Or->setFlags(Flags); 4642 4643 MI.eraseFromParent(); 4644 return Legalized; 4645 } 4646 4647 LegalizerHelper::LegalizeResult 4648 LegalizerHelper::lowerFMinNumMaxNum(MachineInstr &MI) { 4649 unsigned NewOp = MI.getOpcode() == TargetOpcode::G_FMINNUM ? 4650 TargetOpcode::G_FMINNUM_IEEE : TargetOpcode::G_FMAXNUM_IEEE; 4651 4652 Register Dst = MI.getOperand(0).getReg(); 4653 Register Src0 = MI.getOperand(1).getReg(); 4654 Register Src1 = MI.getOperand(2).getReg(); 4655 LLT Ty = MRI.getType(Dst); 4656 4657 if (!MI.getFlag(MachineInstr::FmNoNans)) { 4658 // Insert canonicalizes if it's possible we need to quiet to get correct 4659 // sNaN behavior. 4660 4661 // Note this must be done here, and not as an optimization combine in the 4662 // absence of a dedicate quiet-snan instruction as we're using an 4663 // omni-purpose G_FCANONICALIZE. 4664 if (!isKnownNeverSNaN(Src0, MRI)) 4665 Src0 = MIRBuilder.buildFCanonicalize(Ty, Src0, MI.getFlags()).getReg(0); 4666 4667 if (!isKnownNeverSNaN(Src1, MRI)) 4668 Src1 = MIRBuilder.buildFCanonicalize(Ty, Src1, MI.getFlags()).getReg(0); 4669 } 4670 4671 // If there are no nans, it's safe to simply replace this with the non-IEEE 4672 // version. 4673 MIRBuilder.buildInstr(NewOp, {Dst}, {Src0, Src1}, MI.getFlags()); 4674 MI.eraseFromParent(); 4675 return Legalized; 4676 } 4677 4678 LegalizerHelper::LegalizeResult LegalizerHelper::lowerFMad(MachineInstr &MI) { 4679 // Expand G_FMAD a, b, c -> G_FADD (G_FMUL a, b), c 4680 Register DstReg = MI.getOperand(0).getReg(); 4681 LLT Ty = MRI.getType(DstReg); 4682 unsigned Flags = MI.getFlags(); 4683 4684 auto Mul = MIRBuilder.buildFMul(Ty, MI.getOperand(1), MI.getOperand(2), 4685 Flags); 4686 MIRBuilder.buildFAdd(DstReg, Mul, MI.getOperand(3), Flags); 4687 MI.eraseFromParent(); 4688 return Legalized; 4689 } 4690 4691 LegalizerHelper::LegalizeResult 4692 LegalizerHelper::lowerIntrinsicRound(MachineInstr &MI) { 4693 Register DstReg = MI.getOperand(0).getReg(); 4694 Register SrcReg = MI.getOperand(1).getReg(); 4695 unsigned Flags = MI.getFlags(); 4696 LLT Ty = MRI.getType(DstReg); 4697 const LLT CondTy = Ty.changeElementSize(1); 4698 4699 // result = trunc(src); 4700 // if (src < 0.0 && src != result) 4701 // result += -1.0. 4702 4703 auto Zero = MIRBuilder.buildFConstant(Ty, 0.0); 4704 auto Trunc = MIRBuilder.buildIntrinsicTrunc(Ty, SrcReg, Flags); 4705 4706 auto Lt0 = MIRBuilder.buildFCmp(CmpInst::FCMP_OLT, CondTy, 4707 SrcReg, Zero, Flags); 4708 auto NeTrunc = MIRBuilder.buildFCmp(CmpInst::FCMP_ONE, CondTy, 4709 SrcReg, Trunc, Flags); 4710 auto And = MIRBuilder.buildAnd(CondTy, Lt0, NeTrunc); 4711 auto AddVal = MIRBuilder.buildSITOFP(Ty, And); 4712 4713 MIRBuilder.buildFAdd(DstReg, Trunc, AddVal); 4714 MI.eraseFromParent(); 4715 return Legalized; 4716 } 4717 4718 LegalizerHelper::LegalizeResult 4719 LegalizerHelper::lowerUnmergeValues(MachineInstr &MI) { 4720 const unsigned NumDst = MI.getNumOperands() - 1; 4721 const Register SrcReg = MI.getOperand(NumDst).getReg(); 4722 LLT SrcTy = MRI.getType(SrcReg); 4723 4724 Register Dst0Reg = MI.getOperand(0).getReg(); 4725 LLT DstTy = MRI.getType(Dst0Reg); 4726 4727 4728 // Expand scalarizing unmerge as bitcast to integer and shift. 4729 if (!DstTy.isVector() && SrcTy.isVector() && 4730 SrcTy.getElementType() == DstTy) { 4731 LLT IntTy = LLT::scalar(SrcTy.getSizeInBits()); 4732 Register Cast = MIRBuilder.buildBitcast(IntTy, SrcReg).getReg(0); 4733 4734 MIRBuilder.buildTrunc(Dst0Reg, Cast); 4735 4736 const unsigned DstSize = DstTy.getSizeInBits(); 4737 unsigned Offset = DstSize; 4738 for (unsigned I = 1; I != NumDst; ++I, Offset += DstSize) { 4739 auto ShiftAmt = MIRBuilder.buildConstant(IntTy, Offset); 4740 auto Shift = MIRBuilder.buildLShr(IntTy, Cast, ShiftAmt); 4741 MIRBuilder.buildTrunc(MI.getOperand(I), Shift); 4742 } 4743 4744 MI.eraseFromParent(); 4745 return Legalized; 4746 } 4747 4748 return UnableToLegalize; 4749 } 4750 4751 LegalizerHelper::LegalizeResult 4752 LegalizerHelper::lowerShuffleVector(MachineInstr &MI) { 4753 Register DstReg = MI.getOperand(0).getReg(); 4754 Register Src0Reg = MI.getOperand(1).getReg(); 4755 Register Src1Reg = MI.getOperand(2).getReg(); 4756 LLT Src0Ty = MRI.getType(Src0Reg); 4757 LLT DstTy = MRI.getType(DstReg); 4758 LLT IdxTy = LLT::scalar(32); 4759 4760 ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask(); 4761 4762 if (DstTy.isScalar()) { 4763 if (Src0Ty.isVector()) 4764 return UnableToLegalize; 4765 4766 // This is just a SELECT. 4767 assert(Mask.size() == 1 && "Expected a single mask element"); 4768 Register Val; 4769 if (Mask[0] < 0 || Mask[0] > 1) 4770 Val = MIRBuilder.buildUndef(DstTy).getReg(0); 4771 else 4772 Val = Mask[0] == 0 ? Src0Reg : Src1Reg; 4773 MIRBuilder.buildCopy(DstReg, Val); 4774 MI.eraseFromParent(); 4775 return Legalized; 4776 } 4777 4778 Register Undef; 4779 SmallVector<Register, 32> BuildVec; 4780 LLT EltTy = DstTy.getElementType(); 4781 4782 for (int Idx : Mask) { 4783 if (Idx < 0) { 4784 if (!Undef.isValid()) 4785 Undef = MIRBuilder.buildUndef(EltTy).getReg(0); 4786 BuildVec.push_back(Undef); 4787 continue; 4788 } 4789 4790 if (Src0Ty.isScalar()) { 4791 BuildVec.push_back(Idx == 0 ? Src0Reg : Src1Reg); 4792 } else { 4793 int NumElts = Src0Ty.getNumElements(); 4794 Register SrcVec = Idx < NumElts ? Src0Reg : Src1Reg; 4795 int ExtractIdx = Idx < NumElts ? Idx : Idx - NumElts; 4796 auto IdxK = MIRBuilder.buildConstant(IdxTy, ExtractIdx); 4797 auto Extract = MIRBuilder.buildExtractVectorElement(EltTy, SrcVec, IdxK); 4798 BuildVec.push_back(Extract.getReg(0)); 4799 } 4800 } 4801 4802 MIRBuilder.buildBuildVector(DstReg, BuildVec); 4803 MI.eraseFromParent(); 4804 return Legalized; 4805 } 4806 4807 LegalizerHelper::LegalizeResult 4808 LegalizerHelper::lowerDynStackAlloc(MachineInstr &MI) { 4809 Register Dst = MI.getOperand(0).getReg(); 4810 Register AllocSize = MI.getOperand(1).getReg(); 4811 unsigned Align = MI.getOperand(2).getImm(); 4812 4813 const auto &MF = *MI.getMF(); 4814 const auto &TLI = *MF.getSubtarget().getTargetLowering(); 4815 4816 LLT PtrTy = MRI.getType(Dst); 4817 LLT IntPtrTy = LLT::scalar(PtrTy.getSizeInBits()); 4818 4819 Register SPReg = TLI.getStackPointerRegisterToSaveRestore(); 4820 auto SPTmp = MIRBuilder.buildCopy(PtrTy, SPReg); 4821 SPTmp = MIRBuilder.buildCast(IntPtrTy, SPTmp); 4822 4823 // Subtract the final alloc from the SP. We use G_PTRTOINT here so we don't 4824 // have to generate an extra instruction to negate the alloc and then use 4825 // G_PTR_ADD to add the negative offset. 4826 auto Alloc = MIRBuilder.buildSub(IntPtrTy, SPTmp, AllocSize); 4827 if (Align) { 4828 APInt AlignMask(IntPtrTy.getSizeInBits(), Align, true); 4829 AlignMask.negate(); 4830 auto AlignCst = MIRBuilder.buildConstant(IntPtrTy, AlignMask); 4831 Alloc = MIRBuilder.buildAnd(IntPtrTy, Alloc, AlignCst); 4832 } 4833 4834 SPTmp = MIRBuilder.buildCast(PtrTy, Alloc); 4835 MIRBuilder.buildCopy(SPReg, SPTmp); 4836 MIRBuilder.buildCopy(Dst, SPTmp); 4837 4838 MI.eraseFromParent(); 4839 return Legalized; 4840 } 4841 4842 LegalizerHelper::LegalizeResult 4843 LegalizerHelper::lowerExtract(MachineInstr &MI) { 4844 Register Dst = MI.getOperand(0).getReg(); 4845 Register Src = MI.getOperand(1).getReg(); 4846 unsigned Offset = MI.getOperand(2).getImm(); 4847 4848 LLT DstTy = MRI.getType(Dst); 4849 LLT SrcTy = MRI.getType(Src); 4850 4851 if (DstTy.isScalar() && 4852 (SrcTy.isScalar() || 4853 (SrcTy.isVector() && DstTy == SrcTy.getElementType()))) { 4854 LLT SrcIntTy = SrcTy; 4855 if (!SrcTy.isScalar()) { 4856 SrcIntTy = LLT::scalar(SrcTy.getSizeInBits()); 4857 Src = MIRBuilder.buildBitcast(SrcIntTy, Src).getReg(0); 4858 } 4859 4860 if (Offset == 0) 4861 MIRBuilder.buildTrunc(Dst, Src); 4862 else { 4863 auto ShiftAmt = MIRBuilder.buildConstant(SrcIntTy, Offset); 4864 auto Shr = MIRBuilder.buildLShr(SrcIntTy, Src, ShiftAmt); 4865 MIRBuilder.buildTrunc(Dst, Shr); 4866 } 4867 4868 MI.eraseFromParent(); 4869 return Legalized; 4870 } 4871 4872 return UnableToLegalize; 4873 } 4874 4875 LegalizerHelper::LegalizeResult LegalizerHelper::lowerInsert(MachineInstr &MI) { 4876 Register Dst = MI.getOperand(0).getReg(); 4877 Register Src = MI.getOperand(1).getReg(); 4878 Register InsertSrc = MI.getOperand(2).getReg(); 4879 uint64_t Offset = MI.getOperand(3).getImm(); 4880 4881 LLT DstTy = MRI.getType(Src); 4882 LLT InsertTy = MRI.getType(InsertSrc); 4883 4884 if (InsertTy.isScalar() && 4885 (DstTy.isScalar() || 4886 (DstTy.isVector() && DstTy.getElementType() == InsertTy))) { 4887 LLT IntDstTy = DstTy; 4888 if (!DstTy.isScalar()) { 4889 IntDstTy = LLT::scalar(DstTy.getSizeInBits()); 4890 Src = MIRBuilder.buildBitcast(IntDstTy, Src).getReg(0); 4891 } 4892 4893 Register ExtInsSrc = MIRBuilder.buildZExt(IntDstTy, InsertSrc).getReg(0); 4894 if (Offset != 0) { 4895 auto ShiftAmt = MIRBuilder.buildConstant(IntDstTy, Offset); 4896 ExtInsSrc = MIRBuilder.buildShl(IntDstTy, ExtInsSrc, ShiftAmt).getReg(0); 4897 } 4898 4899 APInt MaskVal = APInt::getBitsSetWithWrap(DstTy.getSizeInBits(), 4900 Offset + InsertTy.getSizeInBits(), 4901 Offset); 4902 4903 auto Mask = MIRBuilder.buildConstant(IntDstTy, MaskVal); 4904 auto MaskedSrc = MIRBuilder.buildAnd(IntDstTy, Src, Mask); 4905 auto Or = MIRBuilder.buildOr(IntDstTy, MaskedSrc, ExtInsSrc); 4906 4907 MIRBuilder.buildBitcast(Dst, Or); 4908 MI.eraseFromParent(); 4909 return Legalized; 4910 } 4911 4912 return UnableToLegalize; 4913 } 4914 4915 LegalizerHelper::LegalizeResult 4916 LegalizerHelper::lowerSADDO_SSUBO(MachineInstr &MI) { 4917 Register Dst0 = MI.getOperand(0).getReg(); 4918 Register Dst1 = MI.getOperand(1).getReg(); 4919 Register LHS = MI.getOperand(2).getReg(); 4920 Register RHS = MI.getOperand(3).getReg(); 4921 const bool IsAdd = MI.getOpcode() == TargetOpcode::G_SADDO; 4922 4923 LLT Ty = MRI.getType(Dst0); 4924 LLT BoolTy = MRI.getType(Dst1); 4925 4926 if (IsAdd) 4927 MIRBuilder.buildAdd(Dst0, LHS, RHS); 4928 else 4929 MIRBuilder.buildSub(Dst0, LHS, RHS); 4930 4931 // TODO: If SADDSAT/SSUBSAT is legal, compare results to detect overflow. 4932 4933 auto Zero = MIRBuilder.buildConstant(Ty, 0); 4934 4935 // For an addition, the result should be less than one of the operands (LHS) 4936 // if and only if the other operand (RHS) is negative, otherwise there will 4937 // be overflow. 4938 // For a subtraction, the result should be less than one of the operands 4939 // (LHS) if and only if the other operand (RHS) is (non-zero) positive, 4940 // otherwise there will be overflow. 4941 auto ResultLowerThanLHS = 4942 MIRBuilder.buildICmp(CmpInst::ICMP_SLT, BoolTy, Dst0, LHS); 4943 auto ConditionRHS = MIRBuilder.buildICmp( 4944 IsAdd ? CmpInst::ICMP_SLT : CmpInst::ICMP_SGT, BoolTy, RHS, Zero); 4945 4946 MIRBuilder.buildXor(Dst1, ConditionRHS, ResultLowerThanLHS); 4947 MI.eraseFromParent(); 4948 return Legalized; 4949 } 4950 4951 LegalizerHelper::LegalizeResult 4952 LegalizerHelper::lowerBswap(MachineInstr &MI) { 4953 Register Dst = MI.getOperand(0).getReg(); 4954 Register Src = MI.getOperand(1).getReg(); 4955 const LLT Ty = MRI.getType(Src); 4956 unsigned SizeInBytes = Ty.getSizeInBytes(); 4957 unsigned BaseShiftAmt = (SizeInBytes - 1) * 8; 4958 4959 // Swap most and least significant byte, set remaining bytes in Res to zero. 4960 auto ShiftAmt = MIRBuilder.buildConstant(Ty, BaseShiftAmt); 4961 auto LSByteShiftedLeft = MIRBuilder.buildShl(Ty, Src, ShiftAmt); 4962 auto MSByteShiftedRight = MIRBuilder.buildLShr(Ty, Src, ShiftAmt); 4963 auto Res = MIRBuilder.buildOr(Ty, MSByteShiftedRight, LSByteShiftedLeft); 4964 4965 // Set i-th high/low byte in Res to i-th low/high byte from Src. 4966 for (unsigned i = 1; i < SizeInBytes / 2; ++i) { 4967 // AND with Mask leaves byte i unchanged and sets remaining bytes to 0. 4968 APInt APMask(SizeInBytes * 8, 0xFF << (i * 8)); 4969 auto Mask = MIRBuilder.buildConstant(Ty, APMask); 4970 auto ShiftAmt = MIRBuilder.buildConstant(Ty, BaseShiftAmt - 16 * i); 4971 // Low byte shifted left to place of high byte: (Src & Mask) << ShiftAmt. 4972 auto LoByte = MIRBuilder.buildAnd(Ty, Src, Mask); 4973 auto LoShiftedLeft = MIRBuilder.buildShl(Ty, LoByte, ShiftAmt); 4974 Res = MIRBuilder.buildOr(Ty, Res, LoShiftedLeft); 4975 // High byte shifted right to place of low byte: (Src >> ShiftAmt) & Mask. 4976 auto SrcShiftedRight = MIRBuilder.buildLShr(Ty, Src, ShiftAmt); 4977 auto HiShiftedRight = MIRBuilder.buildAnd(Ty, SrcShiftedRight, Mask); 4978 Res = MIRBuilder.buildOr(Ty, Res, HiShiftedRight); 4979 } 4980 Res.getInstr()->getOperand(0).setReg(Dst); 4981 4982 MI.eraseFromParent(); 4983 return Legalized; 4984 } 4985 4986 //{ (Src & Mask) >> N } | { (Src << N) & Mask } 4987 static MachineInstrBuilder SwapN(unsigned N, DstOp Dst, MachineIRBuilder &B, 4988 MachineInstrBuilder Src, APInt Mask) { 4989 const LLT Ty = Dst.getLLTTy(*B.getMRI()); 4990 MachineInstrBuilder C_N = B.buildConstant(Ty, N); 4991 MachineInstrBuilder MaskLoNTo0 = B.buildConstant(Ty, Mask); 4992 auto LHS = B.buildLShr(Ty, B.buildAnd(Ty, Src, MaskLoNTo0), C_N); 4993 auto RHS = B.buildAnd(Ty, B.buildShl(Ty, Src, C_N), MaskLoNTo0); 4994 return B.buildOr(Dst, LHS, RHS); 4995 } 4996 4997 LegalizerHelper::LegalizeResult 4998 LegalizerHelper::lowerBitreverse(MachineInstr &MI) { 4999 Register Dst = MI.getOperand(0).getReg(); 5000 Register Src = MI.getOperand(1).getReg(); 5001 const LLT Ty = MRI.getType(Src); 5002 unsigned Size = Ty.getSizeInBits(); 5003 5004 MachineInstrBuilder BSWAP = 5005 MIRBuilder.buildInstr(TargetOpcode::G_BSWAP, {Ty}, {Src}); 5006 5007 // swap high and low 4 bits in 8 bit blocks 7654|3210 -> 3210|7654 5008 // [(val & 0xF0F0F0F0) >> 4] | [(val & 0x0F0F0F0F) << 4] 5009 // -> [(val & 0xF0F0F0F0) >> 4] | [(val << 4) & 0xF0F0F0F0] 5010 MachineInstrBuilder Swap4 = 5011 SwapN(4, Ty, MIRBuilder, BSWAP, APInt::getSplat(Size, APInt(8, 0xF0))); 5012 5013 // swap high and low 2 bits in 4 bit blocks 32|10 76|54 -> 10|32 54|76 5014 // [(val & 0xCCCCCCCC) >> 2] & [(val & 0x33333333) << 2] 5015 // -> [(val & 0xCCCCCCCC) >> 2] & [(val << 2) & 0xCCCCCCCC] 5016 MachineInstrBuilder Swap2 = 5017 SwapN(2, Ty, MIRBuilder, Swap4, APInt::getSplat(Size, APInt(8, 0xCC))); 5018 5019 // swap high and low 1 bit in 2 bit blocks 1|0 3|2 5|4 7|6 -> 0|1 2|3 4|5 6|7 5020 // [(val & 0xAAAAAAAA) >> 1] & [(val & 0x55555555) << 1] 5021 // -> [(val & 0xAAAAAAAA) >> 1] & [(val << 1) & 0xAAAAAAAA] 5022 SwapN(1, Dst, MIRBuilder, Swap2, APInt::getSplat(Size, APInt(8, 0xAA))); 5023 5024 MI.eraseFromParent(); 5025 return Legalized; 5026 } 5027 5028 LegalizerHelper::LegalizeResult 5029 LegalizerHelper::lowerReadWriteRegister(MachineInstr &MI) { 5030 MachineFunction &MF = MIRBuilder.getMF(); 5031 const TargetSubtargetInfo &STI = MF.getSubtarget(); 5032 const TargetLowering *TLI = STI.getTargetLowering(); 5033 5034 bool IsRead = MI.getOpcode() == TargetOpcode::G_READ_REGISTER; 5035 int NameOpIdx = IsRead ? 1 : 0; 5036 int ValRegIndex = IsRead ? 0 : 1; 5037 5038 Register ValReg = MI.getOperand(ValRegIndex).getReg(); 5039 const LLT Ty = MRI.getType(ValReg); 5040 const MDString *RegStr = cast<MDString>( 5041 cast<MDNode>(MI.getOperand(NameOpIdx).getMetadata())->getOperand(0)); 5042 5043 Register PhysReg = TLI->getRegisterByName(RegStr->getString().data(), Ty, MF); 5044 if (!PhysReg.isValid()) 5045 return UnableToLegalize; 5046 5047 if (IsRead) 5048 MIRBuilder.buildCopy(ValReg, PhysReg); 5049 else 5050 MIRBuilder.buildCopy(PhysReg, ValReg); 5051 5052 MI.eraseFromParent(); 5053 return Legalized; 5054 } 5055