1 //===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file This file implements the LegalizerHelper class to legalize 10 /// individual instructions and the LegalizeMachineIR wrapper pass for the 11 /// primary legalization. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h" 16 #include "llvm/CodeGen/GlobalISel/CallLowering.h" 17 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" 18 #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" 19 #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" 20 #include "llvm/CodeGen/MachineRegisterInfo.h" 21 #include "llvm/CodeGen/TargetFrameLowering.h" 22 #include "llvm/CodeGen/TargetInstrInfo.h" 23 #include "llvm/CodeGen/TargetLowering.h" 24 #include "llvm/CodeGen/TargetSubtargetInfo.h" 25 #include "llvm/Support/Debug.h" 26 #include "llvm/Support/MathExtras.h" 27 #include "llvm/Support/raw_ostream.h" 28 29 #define DEBUG_TYPE "legalizer" 30 31 using namespace llvm; 32 using namespace LegalizeActions; 33 using namespace MIPatternMatch; 34 35 /// Try to break down \p OrigTy into \p NarrowTy sized pieces. 36 /// 37 /// Returns the number of \p NarrowTy elements needed to reconstruct \p OrigTy, 38 /// with any leftover piece as type \p LeftoverTy 39 /// 40 /// Returns -1 in the first element of the pair if the breakdown is not 41 /// satisfiable. 42 static std::pair<int, int> 43 getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) { 44 assert(!LeftoverTy.isValid() && "this is an out argument"); 45 46 unsigned Size = OrigTy.getSizeInBits(); 47 unsigned NarrowSize = NarrowTy.getSizeInBits(); 48 unsigned NumParts = Size / NarrowSize; 49 unsigned LeftoverSize = Size - NumParts * NarrowSize; 50 assert(Size > NarrowSize); 51 52 if (LeftoverSize == 0) 53 return {NumParts, 0}; 54 55 if (NarrowTy.isVector()) { 56 unsigned EltSize = OrigTy.getScalarSizeInBits(); 57 if (LeftoverSize % EltSize != 0) 58 return {-1, -1}; 59 LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize); 60 } else { 61 LeftoverTy = LLT::scalar(LeftoverSize); 62 } 63 64 int NumLeftover = LeftoverSize / LeftoverTy.getSizeInBits(); 65 return std::make_pair(NumParts, NumLeftover); 66 } 67 68 static Type *getFloatTypeForLLT(LLVMContext &Ctx, LLT Ty) { 69 70 if (!Ty.isScalar()) 71 return nullptr; 72 73 switch (Ty.getSizeInBits()) { 74 case 16: 75 return Type::getHalfTy(Ctx); 76 case 32: 77 return Type::getFloatTy(Ctx); 78 case 64: 79 return Type::getDoubleTy(Ctx); 80 case 80: 81 return Type::getX86_FP80Ty(Ctx); 82 case 128: 83 return Type::getFP128Ty(Ctx); 84 default: 85 return nullptr; 86 } 87 } 88 89 LegalizerHelper::LegalizerHelper(MachineFunction &MF, 90 GISelChangeObserver &Observer, 91 MachineIRBuilder &Builder) 92 : MIRBuilder(Builder), Observer(Observer), MRI(MF.getRegInfo()), 93 LI(*MF.getSubtarget().getLegalizerInfo()) { 94 MIRBuilder.setChangeObserver(Observer); 95 } 96 97 LegalizerHelper::LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI, 98 GISelChangeObserver &Observer, 99 MachineIRBuilder &B) 100 : MIRBuilder(B), Observer(Observer), MRI(MF.getRegInfo()), LI(LI) { 101 MIRBuilder.setChangeObserver(Observer); 102 } 103 LegalizerHelper::LegalizeResult 104 LegalizerHelper::legalizeInstrStep(MachineInstr &MI) { 105 LLVM_DEBUG(dbgs() << "Legalizing: " << MI); 106 107 MIRBuilder.setInstrAndDebugLoc(MI); 108 109 if (MI.getOpcode() == TargetOpcode::G_INTRINSIC || 110 MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS) 111 return LI.legalizeIntrinsic(*this, MI) ? Legalized : UnableToLegalize; 112 auto Step = LI.getAction(MI, MRI); 113 switch (Step.Action) { 114 case Legal: 115 LLVM_DEBUG(dbgs() << ".. Already legal\n"); 116 return AlreadyLegal; 117 case Libcall: 118 LLVM_DEBUG(dbgs() << ".. Convert to libcall\n"); 119 return libcall(MI); 120 case NarrowScalar: 121 LLVM_DEBUG(dbgs() << ".. Narrow scalar\n"); 122 return narrowScalar(MI, Step.TypeIdx, Step.NewType); 123 case WidenScalar: 124 LLVM_DEBUG(dbgs() << ".. Widen scalar\n"); 125 return widenScalar(MI, Step.TypeIdx, Step.NewType); 126 case Bitcast: 127 LLVM_DEBUG(dbgs() << ".. Bitcast type\n"); 128 return bitcast(MI, Step.TypeIdx, Step.NewType); 129 case Lower: 130 LLVM_DEBUG(dbgs() << ".. Lower\n"); 131 return lower(MI, Step.TypeIdx, Step.NewType); 132 case FewerElements: 133 LLVM_DEBUG(dbgs() << ".. Reduce number of elements\n"); 134 return fewerElementsVector(MI, Step.TypeIdx, Step.NewType); 135 case MoreElements: 136 LLVM_DEBUG(dbgs() << ".. Increase number of elements\n"); 137 return moreElementsVector(MI, Step.TypeIdx, Step.NewType); 138 case Custom: 139 LLVM_DEBUG(dbgs() << ".. Custom legalization\n"); 140 return LI.legalizeCustom(*this, MI) ? Legalized : UnableToLegalize; 141 default: 142 LLVM_DEBUG(dbgs() << ".. Unable to legalize\n"); 143 return UnableToLegalize; 144 } 145 } 146 147 void LegalizerHelper::extractParts(Register Reg, LLT Ty, int NumParts, 148 SmallVectorImpl<Register> &VRegs) { 149 for (int i = 0; i < NumParts; ++i) 150 VRegs.push_back(MRI.createGenericVirtualRegister(Ty)); 151 MIRBuilder.buildUnmerge(VRegs, Reg); 152 } 153 154 bool LegalizerHelper::extractParts(Register Reg, LLT RegTy, 155 LLT MainTy, LLT &LeftoverTy, 156 SmallVectorImpl<Register> &VRegs, 157 SmallVectorImpl<Register> &LeftoverRegs) { 158 assert(!LeftoverTy.isValid() && "this is an out argument"); 159 160 unsigned RegSize = RegTy.getSizeInBits(); 161 unsigned MainSize = MainTy.getSizeInBits(); 162 unsigned NumParts = RegSize / MainSize; 163 unsigned LeftoverSize = RegSize - NumParts * MainSize; 164 165 // Use an unmerge when possible. 166 if (LeftoverSize == 0) { 167 for (unsigned I = 0; I < NumParts; ++I) 168 VRegs.push_back(MRI.createGenericVirtualRegister(MainTy)); 169 MIRBuilder.buildUnmerge(VRegs, Reg); 170 return true; 171 } 172 173 if (MainTy.isVector()) { 174 unsigned EltSize = MainTy.getScalarSizeInBits(); 175 if (LeftoverSize % EltSize != 0) 176 return false; 177 LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize); 178 } else { 179 LeftoverTy = LLT::scalar(LeftoverSize); 180 } 181 182 // For irregular sizes, extract the individual parts. 183 for (unsigned I = 0; I != NumParts; ++I) { 184 Register NewReg = MRI.createGenericVirtualRegister(MainTy); 185 VRegs.push_back(NewReg); 186 MIRBuilder.buildExtract(NewReg, Reg, MainSize * I); 187 } 188 189 for (unsigned Offset = MainSize * NumParts; Offset < RegSize; 190 Offset += LeftoverSize) { 191 Register NewReg = MRI.createGenericVirtualRegister(LeftoverTy); 192 LeftoverRegs.push_back(NewReg); 193 MIRBuilder.buildExtract(NewReg, Reg, Offset); 194 } 195 196 return true; 197 } 198 199 void LegalizerHelper::insertParts(Register DstReg, 200 LLT ResultTy, LLT PartTy, 201 ArrayRef<Register> PartRegs, 202 LLT LeftoverTy, 203 ArrayRef<Register> LeftoverRegs) { 204 if (!LeftoverTy.isValid()) { 205 assert(LeftoverRegs.empty()); 206 207 if (!ResultTy.isVector()) { 208 MIRBuilder.buildMerge(DstReg, PartRegs); 209 return; 210 } 211 212 if (PartTy.isVector()) 213 MIRBuilder.buildConcatVectors(DstReg, PartRegs); 214 else 215 MIRBuilder.buildBuildVector(DstReg, PartRegs); 216 return; 217 } 218 219 unsigned PartSize = PartTy.getSizeInBits(); 220 unsigned LeftoverPartSize = LeftoverTy.getSizeInBits(); 221 222 Register CurResultReg = MRI.createGenericVirtualRegister(ResultTy); 223 MIRBuilder.buildUndef(CurResultReg); 224 225 unsigned Offset = 0; 226 for (Register PartReg : PartRegs) { 227 Register NewResultReg = MRI.createGenericVirtualRegister(ResultTy); 228 MIRBuilder.buildInsert(NewResultReg, CurResultReg, PartReg, Offset); 229 CurResultReg = NewResultReg; 230 Offset += PartSize; 231 } 232 233 for (unsigned I = 0, E = LeftoverRegs.size(); I != E; ++I) { 234 // Use the original output register for the final insert to avoid a copy. 235 Register NewResultReg = (I + 1 == E) ? 236 DstReg : MRI.createGenericVirtualRegister(ResultTy); 237 238 MIRBuilder.buildInsert(NewResultReg, CurResultReg, LeftoverRegs[I], Offset); 239 CurResultReg = NewResultReg; 240 Offset += LeftoverPartSize; 241 } 242 } 243 244 /// Return the result registers of G_UNMERGE_VALUES \p MI in \p Regs 245 static void getUnmergeResults(SmallVectorImpl<Register> &Regs, 246 const MachineInstr &MI) { 247 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES); 248 249 const int NumResults = MI.getNumOperands() - 1; 250 Regs.resize(NumResults); 251 for (int I = 0; I != NumResults; ++I) 252 Regs[I] = MI.getOperand(I).getReg(); 253 } 254 255 LLT LegalizerHelper::extractGCDType(SmallVectorImpl<Register> &Parts, LLT DstTy, 256 LLT NarrowTy, Register SrcReg) { 257 LLT SrcTy = MRI.getType(SrcReg); 258 259 LLT GCDTy = getGCDType(getGCDType(SrcTy, NarrowTy), DstTy); 260 if (SrcTy == GCDTy) { 261 // If the source already evenly divides the result type, we don't need to do 262 // anything. 263 Parts.push_back(SrcReg); 264 } else { 265 // Need to split into common type sized pieces. 266 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg); 267 getUnmergeResults(Parts, *Unmerge); 268 } 269 270 return GCDTy; 271 } 272 273 LLT LegalizerHelper::buildLCMMergePieces(LLT DstTy, LLT NarrowTy, LLT GCDTy, 274 SmallVectorImpl<Register> &VRegs, 275 unsigned PadStrategy) { 276 LLT LCMTy = getLCMType(DstTy, NarrowTy); 277 278 int NumParts = LCMTy.getSizeInBits() / NarrowTy.getSizeInBits(); 279 int NumSubParts = NarrowTy.getSizeInBits() / GCDTy.getSizeInBits(); 280 int NumOrigSrc = VRegs.size(); 281 282 Register PadReg; 283 284 // Get a value we can use to pad the source value if the sources won't evenly 285 // cover the result type. 286 if (NumOrigSrc < NumParts * NumSubParts) { 287 if (PadStrategy == TargetOpcode::G_ZEXT) 288 PadReg = MIRBuilder.buildConstant(GCDTy, 0).getReg(0); 289 else if (PadStrategy == TargetOpcode::G_ANYEXT) 290 PadReg = MIRBuilder.buildUndef(GCDTy).getReg(0); 291 else { 292 assert(PadStrategy == TargetOpcode::G_SEXT); 293 294 // Shift the sign bit of the low register through the high register. 295 auto ShiftAmt = 296 MIRBuilder.buildConstant(LLT::scalar(64), GCDTy.getSizeInBits() - 1); 297 PadReg = MIRBuilder.buildAShr(GCDTy, VRegs.back(), ShiftAmt).getReg(0); 298 } 299 } 300 301 // Registers for the final merge to be produced. 302 SmallVector<Register, 4> Remerge(NumParts); 303 304 // Registers needed for intermediate merges, which will be merged into a 305 // source for Remerge. 306 SmallVector<Register, 4> SubMerge(NumSubParts); 307 308 // Once we've fully read off the end of the original source bits, we can reuse 309 // the same high bits for remaining padding elements. 310 Register AllPadReg; 311 312 // Build merges to the LCM type to cover the original result type. 313 for (int I = 0; I != NumParts; ++I) { 314 bool AllMergePartsArePadding = true; 315 316 // Build the requested merges to the requested type. 317 for (int J = 0; J != NumSubParts; ++J) { 318 int Idx = I * NumSubParts + J; 319 if (Idx >= NumOrigSrc) { 320 SubMerge[J] = PadReg; 321 continue; 322 } 323 324 SubMerge[J] = VRegs[Idx]; 325 326 // There are meaningful bits here we can't reuse later. 327 AllMergePartsArePadding = false; 328 } 329 330 // If we've filled up a complete piece with padding bits, we can directly 331 // emit the natural sized constant if applicable, rather than a merge of 332 // smaller constants. 333 if (AllMergePartsArePadding && !AllPadReg) { 334 if (PadStrategy == TargetOpcode::G_ANYEXT) 335 AllPadReg = MIRBuilder.buildUndef(NarrowTy).getReg(0); 336 else if (PadStrategy == TargetOpcode::G_ZEXT) 337 AllPadReg = MIRBuilder.buildConstant(NarrowTy, 0).getReg(0); 338 339 // If this is a sign extension, we can't materialize a trivial constant 340 // with the right type and have to produce a merge. 341 } 342 343 if (AllPadReg) { 344 // Avoid creating additional instructions if we're just adding additional 345 // copies of padding bits. 346 Remerge[I] = AllPadReg; 347 continue; 348 } 349 350 if (NumSubParts == 1) 351 Remerge[I] = SubMerge[0]; 352 else 353 Remerge[I] = MIRBuilder.buildMerge(NarrowTy, SubMerge).getReg(0); 354 355 // In the sign extend padding case, re-use the first all-signbit merge. 356 if (AllMergePartsArePadding && !AllPadReg) 357 AllPadReg = Remerge[I]; 358 } 359 360 VRegs = std::move(Remerge); 361 return LCMTy; 362 } 363 364 void LegalizerHelper::buildWidenedRemergeToDst(Register DstReg, LLT LCMTy, 365 ArrayRef<Register> RemergeRegs) { 366 LLT DstTy = MRI.getType(DstReg); 367 368 // Create the merge to the widened source, and extract the relevant bits into 369 // the result. 370 371 if (DstTy == LCMTy) { 372 MIRBuilder.buildMerge(DstReg, RemergeRegs); 373 return; 374 } 375 376 auto Remerge = MIRBuilder.buildMerge(LCMTy, RemergeRegs); 377 if (DstTy.isScalar() && LCMTy.isScalar()) { 378 MIRBuilder.buildTrunc(DstReg, Remerge); 379 return; 380 } 381 382 if (LCMTy.isVector()) { 383 MIRBuilder.buildExtract(DstReg, Remerge, 0); 384 return; 385 } 386 387 llvm_unreachable("unhandled case"); 388 } 389 390 static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) { 391 #define RTLIBCASE_INT(LibcallPrefix) \ 392 do { \ 393 switch (Size) { \ 394 case 32: \ 395 return RTLIB::LibcallPrefix##32; \ 396 case 64: \ 397 return RTLIB::LibcallPrefix##64; \ 398 case 128: \ 399 return RTLIB::LibcallPrefix##128; \ 400 default: \ 401 llvm_unreachable("unexpected size"); \ 402 } \ 403 } while (0) 404 405 #define RTLIBCASE(LibcallPrefix) \ 406 do { \ 407 switch (Size) { \ 408 case 32: \ 409 return RTLIB::LibcallPrefix##32; \ 410 case 64: \ 411 return RTLIB::LibcallPrefix##64; \ 412 case 80: \ 413 return RTLIB::LibcallPrefix##80; \ 414 case 128: \ 415 return RTLIB::LibcallPrefix##128; \ 416 default: \ 417 llvm_unreachable("unexpected size"); \ 418 } \ 419 } while (0) 420 421 switch (Opcode) { 422 case TargetOpcode::G_SDIV: 423 RTLIBCASE_INT(SDIV_I); 424 case TargetOpcode::G_UDIV: 425 RTLIBCASE_INT(UDIV_I); 426 case TargetOpcode::G_SREM: 427 RTLIBCASE_INT(SREM_I); 428 case TargetOpcode::G_UREM: 429 RTLIBCASE_INT(UREM_I); 430 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 431 RTLIBCASE_INT(CTLZ_I); 432 case TargetOpcode::G_FADD: 433 RTLIBCASE(ADD_F); 434 case TargetOpcode::G_FSUB: 435 RTLIBCASE(SUB_F); 436 case TargetOpcode::G_FMUL: 437 RTLIBCASE(MUL_F); 438 case TargetOpcode::G_FDIV: 439 RTLIBCASE(DIV_F); 440 case TargetOpcode::G_FEXP: 441 RTLIBCASE(EXP_F); 442 case TargetOpcode::G_FEXP2: 443 RTLIBCASE(EXP2_F); 444 case TargetOpcode::G_FREM: 445 RTLIBCASE(REM_F); 446 case TargetOpcode::G_FPOW: 447 RTLIBCASE(POW_F); 448 case TargetOpcode::G_FMA: 449 RTLIBCASE(FMA_F); 450 case TargetOpcode::G_FSIN: 451 RTLIBCASE(SIN_F); 452 case TargetOpcode::G_FCOS: 453 RTLIBCASE(COS_F); 454 case TargetOpcode::G_FLOG10: 455 RTLIBCASE(LOG10_F); 456 case TargetOpcode::G_FLOG: 457 RTLIBCASE(LOG_F); 458 case TargetOpcode::G_FLOG2: 459 RTLIBCASE(LOG2_F); 460 case TargetOpcode::G_FCEIL: 461 RTLIBCASE(CEIL_F); 462 case TargetOpcode::G_FFLOOR: 463 RTLIBCASE(FLOOR_F); 464 case TargetOpcode::G_FMINNUM: 465 RTLIBCASE(FMIN_F); 466 case TargetOpcode::G_FMAXNUM: 467 RTLIBCASE(FMAX_F); 468 case TargetOpcode::G_FSQRT: 469 RTLIBCASE(SQRT_F); 470 case TargetOpcode::G_FRINT: 471 RTLIBCASE(RINT_F); 472 case TargetOpcode::G_FNEARBYINT: 473 RTLIBCASE(NEARBYINT_F); 474 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: 475 RTLIBCASE(ROUNDEVEN_F); 476 } 477 llvm_unreachable("Unknown libcall function"); 478 } 479 480 /// True if an instruction is in tail position in its caller. Intended for 481 /// legalizing libcalls as tail calls when possible. 482 static bool isLibCallInTailPosition(const TargetInstrInfo &TII, 483 MachineInstr &MI) { 484 MachineBasicBlock &MBB = *MI.getParent(); 485 const Function &F = MBB.getParent()->getFunction(); 486 487 // Conservatively require the attributes of the call to match those of 488 // the return. Ignore NoAlias and NonNull because they don't affect the 489 // call sequence. 490 AttributeList CallerAttrs = F.getAttributes(); 491 if (AttrBuilder(CallerAttrs, AttributeList::ReturnIndex) 492 .removeAttribute(Attribute::NoAlias) 493 .removeAttribute(Attribute::NonNull) 494 .hasAttributes()) 495 return false; 496 497 // It's not safe to eliminate the sign / zero extension of the return value. 498 if (CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::ZExt) || 499 CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::SExt)) 500 return false; 501 502 // Only tail call if the following instruction is a standard return. 503 auto Next = next_nodbg(MI.getIterator(), MBB.instr_end()); 504 if (Next == MBB.instr_end() || TII.isTailCall(*Next) || !Next->isReturn()) 505 return false; 506 507 return true; 508 } 509 510 LegalizerHelper::LegalizeResult 511 llvm::createLibcall(MachineIRBuilder &MIRBuilder, const char *Name, 512 const CallLowering::ArgInfo &Result, 513 ArrayRef<CallLowering::ArgInfo> Args, 514 const CallingConv::ID CC) { 515 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering(); 516 517 CallLowering::CallLoweringInfo Info; 518 Info.CallConv = CC; 519 Info.Callee = MachineOperand::CreateES(Name); 520 Info.OrigRet = Result; 521 std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs)); 522 if (!CLI.lowerCall(MIRBuilder, Info)) 523 return LegalizerHelper::UnableToLegalize; 524 525 return LegalizerHelper::Legalized; 526 } 527 528 LegalizerHelper::LegalizeResult 529 llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall, 530 const CallLowering::ArgInfo &Result, 531 ArrayRef<CallLowering::ArgInfo> Args) { 532 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering(); 533 const char *Name = TLI.getLibcallName(Libcall); 534 const CallingConv::ID CC = TLI.getLibcallCallingConv(Libcall); 535 return createLibcall(MIRBuilder, Name, Result, Args, CC); 536 } 537 538 // Useful for libcalls where all operands have the same type. 539 static LegalizerHelper::LegalizeResult 540 simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size, 541 Type *OpType) { 542 auto Libcall = getRTLibDesc(MI.getOpcode(), Size); 543 544 SmallVector<CallLowering::ArgInfo, 3> Args; 545 for (unsigned i = 1; i < MI.getNumOperands(); i++) 546 Args.push_back({MI.getOperand(i).getReg(), OpType}); 547 return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), OpType}, 548 Args); 549 } 550 551 LegalizerHelper::LegalizeResult 552 llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, 553 MachineInstr &MI) { 554 assert(MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS); 555 auto &Ctx = MIRBuilder.getMF().getFunction().getContext(); 556 557 SmallVector<CallLowering::ArgInfo, 3> Args; 558 // Add all the args, except for the last which is an imm denoting 'tail'. 559 for (unsigned i = 1; i < MI.getNumOperands() - 1; i++) { 560 Register Reg = MI.getOperand(i).getReg(); 561 562 // Need derive an IR type for call lowering. 563 LLT OpLLT = MRI.getType(Reg); 564 Type *OpTy = nullptr; 565 if (OpLLT.isPointer()) 566 OpTy = Type::getInt8PtrTy(Ctx, OpLLT.getAddressSpace()); 567 else 568 OpTy = IntegerType::get(Ctx, OpLLT.getSizeInBits()); 569 Args.push_back({Reg, OpTy}); 570 } 571 572 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering(); 573 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering(); 574 Intrinsic::ID ID = MI.getOperand(0).getIntrinsicID(); 575 RTLIB::Libcall RTLibcall; 576 switch (ID) { 577 case Intrinsic::memcpy: 578 RTLibcall = RTLIB::MEMCPY; 579 break; 580 case Intrinsic::memset: 581 RTLibcall = RTLIB::MEMSET; 582 break; 583 case Intrinsic::memmove: 584 RTLibcall = RTLIB::MEMMOVE; 585 break; 586 default: 587 return LegalizerHelper::UnableToLegalize; 588 } 589 const char *Name = TLI.getLibcallName(RTLibcall); 590 591 MIRBuilder.setInstrAndDebugLoc(MI); 592 593 CallLowering::CallLoweringInfo Info; 594 Info.CallConv = TLI.getLibcallCallingConv(RTLibcall); 595 Info.Callee = MachineOperand::CreateES(Name); 596 Info.OrigRet = CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx)); 597 Info.IsTailCall = MI.getOperand(MI.getNumOperands() - 1).getImm() == 1 && 598 isLibCallInTailPosition(MIRBuilder.getTII(), MI); 599 600 std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs)); 601 if (!CLI.lowerCall(MIRBuilder, Info)) 602 return LegalizerHelper::UnableToLegalize; 603 604 if (Info.LoweredTailCall) { 605 assert(Info.IsTailCall && "Lowered tail call when it wasn't a tail call?"); 606 // We must have a return following the call (or debug insts) to get past 607 // isLibCallInTailPosition. 608 do { 609 MachineInstr *Next = MI.getNextNode(); 610 assert(Next && (Next->isReturn() || Next->isDebugInstr()) && 611 "Expected instr following MI to be return or debug inst?"); 612 // We lowered a tail call, so the call is now the return from the block. 613 // Delete the old return. 614 Next->eraseFromParent(); 615 } while (MI.getNextNode()); 616 } 617 618 return LegalizerHelper::Legalized; 619 } 620 621 static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType, 622 Type *FromType) { 623 auto ToMVT = MVT::getVT(ToType); 624 auto FromMVT = MVT::getVT(FromType); 625 626 switch (Opcode) { 627 case TargetOpcode::G_FPEXT: 628 return RTLIB::getFPEXT(FromMVT, ToMVT); 629 case TargetOpcode::G_FPTRUNC: 630 return RTLIB::getFPROUND(FromMVT, ToMVT); 631 case TargetOpcode::G_FPTOSI: 632 return RTLIB::getFPTOSINT(FromMVT, ToMVT); 633 case TargetOpcode::G_FPTOUI: 634 return RTLIB::getFPTOUINT(FromMVT, ToMVT); 635 case TargetOpcode::G_SITOFP: 636 return RTLIB::getSINTTOFP(FromMVT, ToMVT); 637 case TargetOpcode::G_UITOFP: 638 return RTLIB::getUINTTOFP(FromMVT, ToMVT); 639 } 640 llvm_unreachable("Unsupported libcall function"); 641 } 642 643 static LegalizerHelper::LegalizeResult 644 conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType, 645 Type *FromType) { 646 RTLIB::Libcall Libcall = getConvRTLibDesc(MI.getOpcode(), ToType, FromType); 647 return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), ToType}, 648 {{MI.getOperand(1).getReg(), FromType}}); 649 } 650 651 LegalizerHelper::LegalizeResult 652 LegalizerHelper::libcall(MachineInstr &MI) { 653 LLT LLTy = MRI.getType(MI.getOperand(0).getReg()); 654 unsigned Size = LLTy.getSizeInBits(); 655 auto &Ctx = MIRBuilder.getMF().getFunction().getContext(); 656 657 switch (MI.getOpcode()) { 658 default: 659 return UnableToLegalize; 660 case TargetOpcode::G_SDIV: 661 case TargetOpcode::G_UDIV: 662 case TargetOpcode::G_SREM: 663 case TargetOpcode::G_UREM: 664 case TargetOpcode::G_CTLZ_ZERO_UNDEF: { 665 Type *HLTy = IntegerType::get(Ctx, Size); 666 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy); 667 if (Status != Legalized) 668 return Status; 669 break; 670 } 671 case TargetOpcode::G_FADD: 672 case TargetOpcode::G_FSUB: 673 case TargetOpcode::G_FMUL: 674 case TargetOpcode::G_FDIV: 675 case TargetOpcode::G_FMA: 676 case TargetOpcode::G_FPOW: 677 case TargetOpcode::G_FREM: 678 case TargetOpcode::G_FCOS: 679 case TargetOpcode::G_FSIN: 680 case TargetOpcode::G_FLOG10: 681 case TargetOpcode::G_FLOG: 682 case TargetOpcode::G_FLOG2: 683 case TargetOpcode::G_FEXP: 684 case TargetOpcode::G_FEXP2: 685 case TargetOpcode::G_FCEIL: 686 case TargetOpcode::G_FFLOOR: 687 case TargetOpcode::G_FMINNUM: 688 case TargetOpcode::G_FMAXNUM: 689 case TargetOpcode::G_FSQRT: 690 case TargetOpcode::G_FRINT: 691 case TargetOpcode::G_FNEARBYINT: 692 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: { 693 Type *HLTy = getFloatTypeForLLT(Ctx, LLTy); 694 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) { 695 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n"); 696 return UnableToLegalize; 697 } 698 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy); 699 if (Status != Legalized) 700 return Status; 701 break; 702 } 703 case TargetOpcode::G_FPEXT: 704 case TargetOpcode::G_FPTRUNC: { 705 Type *FromTy = getFloatTypeForLLT(Ctx, MRI.getType(MI.getOperand(1).getReg())); 706 Type *ToTy = getFloatTypeForLLT(Ctx, MRI.getType(MI.getOperand(0).getReg())); 707 if (!FromTy || !ToTy) 708 return UnableToLegalize; 709 LegalizeResult Status = conversionLibcall(MI, MIRBuilder, ToTy, FromTy ); 710 if (Status != Legalized) 711 return Status; 712 break; 713 } 714 case TargetOpcode::G_FPTOSI: 715 case TargetOpcode::G_FPTOUI: { 716 // FIXME: Support other types 717 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits(); 718 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 719 if ((ToSize != 32 && ToSize != 64) || (FromSize != 32 && FromSize != 64)) 720 return UnableToLegalize; 721 LegalizeResult Status = conversionLibcall( 722 MI, MIRBuilder, 723 ToSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx), 724 FromSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx)); 725 if (Status != Legalized) 726 return Status; 727 break; 728 } 729 case TargetOpcode::G_SITOFP: 730 case TargetOpcode::G_UITOFP: { 731 // FIXME: Support other types 732 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits(); 733 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 734 if ((FromSize != 32 && FromSize != 64) || (ToSize != 32 && ToSize != 64)) 735 return UnableToLegalize; 736 LegalizeResult Status = conversionLibcall( 737 MI, MIRBuilder, 738 ToSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx), 739 FromSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx)); 740 if (Status != Legalized) 741 return Status; 742 break; 743 } 744 } 745 746 MI.eraseFromParent(); 747 return Legalized; 748 } 749 750 LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI, 751 unsigned TypeIdx, 752 LLT NarrowTy) { 753 uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 754 uint64_t NarrowSize = NarrowTy.getSizeInBits(); 755 756 switch (MI.getOpcode()) { 757 default: 758 return UnableToLegalize; 759 case TargetOpcode::G_IMPLICIT_DEF: { 760 Register DstReg = MI.getOperand(0).getReg(); 761 LLT DstTy = MRI.getType(DstReg); 762 763 // If SizeOp0 is not an exact multiple of NarrowSize, emit 764 // G_ANYEXT(G_IMPLICIT_DEF). Cast result to vector if needed. 765 // FIXME: Although this would also be legal for the general case, it causes 766 // a lot of regressions in the emitted code (superfluous COPYs, artifact 767 // combines not being hit). This seems to be a problem related to the 768 // artifact combiner. 769 if (SizeOp0 % NarrowSize != 0) { 770 LLT ImplicitTy = NarrowTy; 771 if (DstTy.isVector()) 772 ImplicitTy = LLT::vector(DstTy.getNumElements(), ImplicitTy); 773 774 Register ImplicitReg = MIRBuilder.buildUndef(ImplicitTy).getReg(0); 775 MIRBuilder.buildAnyExt(DstReg, ImplicitReg); 776 777 MI.eraseFromParent(); 778 return Legalized; 779 } 780 781 int NumParts = SizeOp0 / NarrowSize; 782 783 SmallVector<Register, 2> DstRegs; 784 for (int i = 0; i < NumParts; ++i) 785 DstRegs.push_back(MIRBuilder.buildUndef(NarrowTy).getReg(0)); 786 787 if (DstTy.isVector()) 788 MIRBuilder.buildBuildVector(DstReg, DstRegs); 789 else 790 MIRBuilder.buildMerge(DstReg, DstRegs); 791 MI.eraseFromParent(); 792 return Legalized; 793 } 794 case TargetOpcode::G_CONSTANT: { 795 LLT Ty = MRI.getType(MI.getOperand(0).getReg()); 796 const APInt &Val = MI.getOperand(1).getCImm()->getValue(); 797 unsigned TotalSize = Ty.getSizeInBits(); 798 unsigned NarrowSize = NarrowTy.getSizeInBits(); 799 int NumParts = TotalSize / NarrowSize; 800 801 SmallVector<Register, 4> PartRegs; 802 for (int I = 0; I != NumParts; ++I) { 803 unsigned Offset = I * NarrowSize; 804 auto K = MIRBuilder.buildConstant(NarrowTy, 805 Val.lshr(Offset).trunc(NarrowSize)); 806 PartRegs.push_back(K.getReg(0)); 807 } 808 809 LLT LeftoverTy; 810 unsigned LeftoverBits = TotalSize - NumParts * NarrowSize; 811 SmallVector<Register, 1> LeftoverRegs; 812 if (LeftoverBits != 0) { 813 LeftoverTy = LLT::scalar(LeftoverBits); 814 auto K = MIRBuilder.buildConstant( 815 LeftoverTy, 816 Val.lshr(NumParts * NarrowSize).trunc(LeftoverBits)); 817 LeftoverRegs.push_back(K.getReg(0)); 818 } 819 820 insertParts(MI.getOperand(0).getReg(), 821 Ty, NarrowTy, PartRegs, LeftoverTy, LeftoverRegs); 822 823 MI.eraseFromParent(); 824 return Legalized; 825 } 826 case TargetOpcode::G_SEXT: 827 case TargetOpcode::G_ZEXT: 828 case TargetOpcode::G_ANYEXT: 829 return narrowScalarExt(MI, TypeIdx, NarrowTy); 830 case TargetOpcode::G_TRUNC: { 831 if (TypeIdx != 1) 832 return UnableToLegalize; 833 834 uint64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits(); 835 if (NarrowTy.getSizeInBits() * 2 != SizeOp1) { 836 LLVM_DEBUG(dbgs() << "Can't narrow trunc to type " << NarrowTy << "\n"); 837 return UnableToLegalize; 838 } 839 840 auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1)); 841 MIRBuilder.buildCopy(MI.getOperand(0), Unmerge.getReg(0)); 842 MI.eraseFromParent(); 843 return Legalized; 844 } 845 846 case TargetOpcode::G_FREEZE: 847 return reduceOperationWidth(MI, TypeIdx, NarrowTy); 848 849 case TargetOpcode::G_ADD: { 850 // FIXME: add support for when SizeOp0 isn't an exact multiple of 851 // NarrowSize. 852 if (SizeOp0 % NarrowSize != 0) 853 return UnableToLegalize; 854 // Expand in terms of carry-setting/consuming G_ADDE instructions. 855 int NumParts = SizeOp0 / NarrowTy.getSizeInBits(); 856 857 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs; 858 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs); 859 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs); 860 861 Register CarryIn; 862 for (int i = 0; i < NumParts; ++i) { 863 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy); 864 Register CarryOut = MRI.createGenericVirtualRegister(LLT::scalar(1)); 865 866 if (i == 0) 867 MIRBuilder.buildUAddo(DstReg, CarryOut, Src1Regs[i], Src2Regs[i]); 868 else { 869 MIRBuilder.buildUAdde(DstReg, CarryOut, Src1Regs[i], 870 Src2Regs[i], CarryIn); 871 } 872 873 DstRegs.push_back(DstReg); 874 CarryIn = CarryOut; 875 } 876 Register DstReg = MI.getOperand(0).getReg(); 877 if(MRI.getType(DstReg).isVector()) 878 MIRBuilder.buildBuildVector(DstReg, DstRegs); 879 else 880 MIRBuilder.buildMerge(DstReg, DstRegs); 881 MI.eraseFromParent(); 882 return Legalized; 883 } 884 case TargetOpcode::G_SUB: { 885 // FIXME: add support for when SizeOp0 isn't an exact multiple of 886 // NarrowSize. 887 if (SizeOp0 % NarrowSize != 0) 888 return UnableToLegalize; 889 890 int NumParts = SizeOp0 / NarrowTy.getSizeInBits(); 891 892 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs; 893 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs); 894 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs); 895 896 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy); 897 Register BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1)); 898 MIRBuilder.buildInstr(TargetOpcode::G_USUBO, {DstReg, BorrowOut}, 899 {Src1Regs[0], Src2Regs[0]}); 900 DstRegs.push_back(DstReg); 901 Register BorrowIn = BorrowOut; 902 for (int i = 1; i < NumParts; ++i) { 903 DstReg = MRI.createGenericVirtualRegister(NarrowTy); 904 BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1)); 905 906 MIRBuilder.buildInstr(TargetOpcode::G_USUBE, {DstReg, BorrowOut}, 907 {Src1Regs[i], Src2Regs[i], BorrowIn}); 908 909 DstRegs.push_back(DstReg); 910 BorrowIn = BorrowOut; 911 } 912 MIRBuilder.buildMerge(MI.getOperand(0), DstRegs); 913 MI.eraseFromParent(); 914 return Legalized; 915 } 916 case TargetOpcode::G_MUL: 917 case TargetOpcode::G_UMULH: 918 return narrowScalarMul(MI, NarrowTy); 919 case TargetOpcode::G_EXTRACT: 920 return narrowScalarExtract(MI, TypeIdx, NarrowTy); 921 case TargetOpcode::G_INSERT: 922 return narrowScalarInsert(MI, TypeIdx, NarrowTy); 923 case TargetOpcode::G_LOAD: { 924 auto &MMO = **MI.memoperands_begin(); 925 Register DstReg = MI.getOperand(0).getReg(); 926 LLT DstTy = MRI.getType(DstReg); 927 if (DstTy.isVector()) 928 return UnableToLegalize; 929 930 if (8 * MMO.getSize() != DstTy.getSizeInBits()) { 931 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy); 932 MIRBuilder.buildLoad(TmpReg, MI.getOperand(1), MMO); 933 MIRBuilder.buildAnyExt(DstReg, TmpReg); 934 MI.eraseFromParent(); 935 return Legalized; 936 } 937 938 return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy); 939 } 940 case TargetOpcode::G_ZEXTLOAD: 941 case TargetOpcode::G_SEXTLOAD: { 942 bool ZExt = MI.getOpcode() == TargetOpcode::G_ZEXTLOAD; 943 Register DstReg = MI.getOperand(0).getReg(); 944 Register PtrReg = MI.getOperand(1).getReg(); 945 946 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy); 947 auto &MMO = **MI.memoperands_begin(); 948 if (MMO.getSizeInBits() == NarrowSize) { 949 MIRBuilder.buildLoad(TmpReg, PtrReg, MMO); 950 } else { 951 MIRBuilder.buildLoadInstr(MI.getOpcode(), TmpReg, PtrReg, MMO); 952 } 953 954 if (ZExt) 955 MIRBuilder.buildZExt(DstReg, TmpReg); 956 else 957 MIRBuilder.buildSExt(DstReg, TmpReg); 958 959 MI.eraseFromParent(); 960 return Legalized; 961 } 962 case TargetOpcode::G_STORE: { 963 const auto &MMO = **MI.memoperands_begin(); 964 965 Register SrcReg = MI.getOperand(0).getReg(); 966 LLT SrcTy = MRI.getType(SrcReg); 967 if (SrcTy.isVector()) 968 return UnableToLegalize; 969 970 int NumParts = SizeOp0 / NarrowSize; 971 unsigned HandledSize = NumParts * NarrowTy.getSizeInBits(); 972 unsigned LeftoverBits = SrcTy.getSizeInBits() - HandledSize; 973 if (SrcTy.isVector() && LeftoverBits != 0) 974 return UnableToLegalize; 975 976 if (8 * MMO.getSize() != SrcTy.getSizeInBits()) { 977 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy); 978 auto &MMO = **MI.memoperands_begin(); 979 MIRBuilder.buildTrunc(TmpReg, SrcReg); 980 MIRBuilder.buildStore(TmpReg, MI.getOperand(1), MMO); 981 MI.eraseFromParent(); 982 return Legalized; 983 } 984 985 return reduceLoadStoreWidth(MI, 0, NarrowTy); 986 } 987 case TargetOpcode::G_SELECT: 988 return narrowScalarSelect(MI, TypeIdx, NarrowTy); 989 case TargetOpcode::G_AND: 990 case TargetOpcode::G_OR: 991 case TargetOpcode::G_XOR: { 992 // Legalize bitwise operation: 993 // A = BinOp<Ty> B, C 994 // into: 995 // B1, ..., BN = G_UNMERGE_VALUES B 996 // C1, ..., CN = G_UNMERGE_VALUES C 997 // A1 = BinOp<Ty/N> B1, C2 998 // ... 999 // AN = BinOp<Ty/N> BN, CN 1000 // A = G_MERGE_VALUES A1, ..., AN 1001 return narrowScalarBasic(MI, TypeIdx, NarrowTy); 1002 } 1003 case TargetOpcode::G_SHL: 1004 case TargetOpcode::G_LSHR: 1005 case TargetOpcode::G_ASHR: 1006 return narrowScalarShift(MI, TypeIdx, NarrowTy); 1007 case TargetOpcode::G_CTLZ: 1008 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 1009 case TargetOpcode::G_CTTZ: 1010 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 1011 case TargetOpcode::G_CTPOP: 1012 if (TypeIdx == 1) 1013 switch (MI.getOpcode()) { 1014 case TargetOpcode::G_CTLZ: 1015 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 1016 return narrowScalarCTLZ(MI, TypeIdx, NarrowTy); 1017 case TargetOpcode::G_CTTZ: 1018 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 1019 return narrowScalarCTTZ(MI, TypeIdx, NarrowTy); 1020 case TargetOpcode::G_CTPOP: 1021 return narrowScalarCTPOP(MI, TypeIdx, NarrowTy); 1022 default: 1023 return UnableToLegalize; 1024 } 1025 1026 Observer.changingInstr(MI); 1027 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT); 1028 Observer.changedInstr(MI); 1029 return Legalized; 1030 case TargetOpcode::G_INTTOPTR: 1031 if (TypeIdx != 1) 1032 return UnableToLegalize; 1033 1034 Observer.changingInstr(MI); 1035 narrowScalarSrc(MI, NarrowTy, 1); 1036 Observer.changedInstr(MI); 1037 return Legalized; 1038 case TargetOpcode::G_PTRTOINT: 1039 if (TypeIdx != 0) 1040 return UnableToLegalize; 1041 1042 Observer.changingInstr(MI); 1043 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT); 1044 Observer.changedInstr(MI); 1045 return Legalized; 1046 case TargetOpcode::G_PHI: { 1047 unsigned NumParts = SizeOp0 / NarrowSize; 1048 SmallVector<Register, 2> DstRegs(NumParts); 1049 SmallVector<SmallVector<Register, 2>, 2> SrcRegs(MI.getNumOperands() / 2); 1050 Observer.changingInstr(MI); 1051 for (unsigned i = 1; i < MI.getNumOperands(); i += 2) { 1052 MachineBasicBlock &OpMBB = *MI.getOperand(i + 1).getMBB(); 1053 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator()); 1054 extractParts(MI.getOperand(i).getReg(), NarrowTy, NumParts, 1055 SrcRegs[i / 2]); 1056 } 1057 MachineBasicBlock &MBB = *MI.getParent(); 1058 MIRBuilder.setInsertPt(MBB, MI); 1059 for (unsigned i = 0; i < NumParts; ++i) { 1060 DstRegs[i] = MRI.createGenericVirtualRegister(NarrowTy); 1061 MachineInstrBuilder MIB = 1062 MIRBuilder.buildInstr(TargetOpcode::G_PHI).addDef(DstRegs[i]); 1063 for (unsigned j = 1; j < MI.getNumOperands(); j += 2) 1064 MIB.addUse(SrcRegs[j / 2][i]).add(MI.getOperand(j + 1)); 1065 } 1066 MIRBuilder.setInsertPt(MBB, MBB.getFirstNonPHI()); 1067 MIRBuilder.buildMerge(MI.getOperand(0), DstRegs); 1068 Observer.changedInstr(MI); 1069 MI.eraseFromParent(); 1070 return Legalized; 1071 } 1072 case TargetOpcode::G_EXTRACT_VECTOR_ELT: 1073 case TargetOpcode::G_INSERT_VECTOR_ELT: { 1074 if (TypeIdx != 2) 1075 return UnableToLegalize; 1076 1077 int OpIdx = MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3; 1078 Observer.changingInstr(MI); 1079 narrowScalarSrc(MI, NarrowTy, OpIdx); 1080 Observer.changedInstr(MI); 1081 return Legalized; 1082 } 1083 case TargetOpcode::G_ICMP: { 1084 uint64_t SrcSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits(); 1085 if (NarrowSize * 2 != SrcSize) 1086 return UnableToLegalize; 1087 1088 Observer.changingInstr(MI); 1089 Register LHSL = MRI.createGenericVirtualRegister(NarrowTy); 1090 Register LHSH = MRI.createGenericVirtualRegister(NarrowTy); 1091 MIRBuilder.buildUnmerge({LHSL, LHSH}, MI.getOperand(2)); 1092 1093 Register RHSL = MRI.createGenericVirtualRegister(NarrowTy); 1094 Register RHSH = MRI.createGenericVirtualRegister(NarrowTy); 1095 MIRBuilder.buildUnmerge({RHSL, RHSH}, MI.getOperand(3)); 1096 1097 CmpInst::Predicate Pred = 1098 static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate()); 1099 LLT ResTy = MRI.getType(MI.getOperand(0).getReg()); 1100 1101 if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE) { 1102 MachineInstrBuilder XorL = MIRBuilder.buildXor(NarrowTy, LHSL, RHSL); 1103 MachineInstrBuilder XorH = MIRBuilder.buildXor(NarrowTy, LHSH, RHSH); 1104 MachineInstrBuilder Or = MIRBuilder.buildOr(NarrowTy, XorL, XorH); 1105 MachineInstrBuilder Zero = MIRBuilder.buildConstant(NarrowTy, 0); 1106 MIRBuilder.buildICmp(Pred, MI.getOperand(0), Or, Zero); 1107 } else { 1108 MachineInstrBuilder CmpH = MIRBuilder.buildICmp(Pred, ResTy, LHSH, RHSH); 1109 MachineInstrBuilder CmpHEQ = 1110 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, ResTy, LHSH, RHSH); 1111 MachineInstrBuilder CmpLU = MIRBuilder.buildICmp( 1112 ICmpInst::getUnsignedPredicate(Pred), ResTy, LHSL, RHSL); 1113 MIRBuilder.buildSelect(MI.getOperand(0), CmpHEQ, CmpLU, CmpH); 1114 } 1115 Observer.changedInstr(MI); 1116 MI.eraseFromParent(); 1117 return Legalized; 1118 } 1119 case TargetOpcode::G_SEXT_INREG: { 1120 if (TypeIdx != 0) 1121 return UnableToLegalize; 1122 1123 int64_t SizeInBits = MI.getOperand(2).getImm(); 1124 1125 // So long as the new type has more bits than the bits we're extending we 1126 // don't need to break it apart. 1127 if (NarrowTy.getScalarSizeInBits() >= SizeInBits) { 1128 Observer.changingInstr(MI); 1129 // We don't lose any non-extension bits by truncating the src and 1130 // sign-extending the dst. 1131 MachineOperand &MO1 = MI.getOperand(1); 1132 auto TruncMIB = MIRBuilder.buildTrunc(NarrowTy, MO1); 1133 MO1.setReg(TruncMIB.getReg(0)); 1134 1135 MachineOperand &MO2 = MI.getOperand(0); 1136 Register DstExt = MRI.createGenericVirtualRegister(NarrowTy); 1137 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); 1138 MIRBuilder.buildSExt(MO2, DstExt); 1139 MO2.setReg(DstExt); 1140 Observer.changedInstr(MI); 1141 return Legalized; 1142 } 1143 1144 // Break it apart. Components below the extension point are unmodified. The 1145 // component containing the extension point becomes a narrower SEXT_INREG. 1146 // Components above it are ashr'd from the component containing the 1147 // extension point. 1148 if (SizeOp0 % NarrowSize != 0) 1149 return UnableToLegalize; 1150 int NumParts = SizeOp0 / NarrowSize; 1151 1152 // List the registers where the destination will be scattered. 1153 SmallVector<Register, 2> DstRegs; 1154 // List the registers where the source will be split. 1155 SmallVector<Register, 2> SrcRegs; 1156 1157 // Create all the temporary registers. 1158 for (int i = 0; i < NumParts; ++i) { 1159 Register SrcReg = MRI.createGenericVirtualRegister(NarrowTy); 1160 1161 SrcRegs.push_back(SrcReg); 1162 } 1163 1164 // Explode the big arguments into smaller chunks. 1165 MIRBuilder.buildUnmerge(SrcRegs, MI.getOperand(1)); 1166 1167 Register AshrCstReg = 1168 MIRBuilder.buildConstant(NarrowTy, NarrowTy.getScalarSizeInBits() - 1) 1169 .getReg(0); 1170 Register FullExtensionReg = 0; 1171 Register PartialExtensionReg = 0; 1172 1173 // Do the operation on each small part. 1174 for (int i = 0; i < NumParts; ++i) { 1175 if ((i + 1) * NarrowTy.getScalarSizeInBits() < SizeInBits) 1176 DstRegs.push_back(SrcRegs[i]); 1177 else if (i * NarrowTy.getScalarSizeInBits() > SizeInBits) { 1178 assert(PartialExtensionReg && 1179 "Expected to visit partial extension before full"); 1180 if (FullExtensionReg) { 1181 DstRegs.push_back(FullExtensionReg); 1182 continue; 1183 } 1184 DstRegs.push_back( 1185 MIRBuilder.buildAShr(NarrowTy, PartialExtensionReg, AshrCstReg) 1186 .getReg(0)); 1187 FullExtensionReg = DstRegs.back(); 1188 } else { 1189 DstRegs.push_back( 1190 MIRBuilder 1191 .buildInstr( 1192 TargetOpcode::G_SEXT_INREG, {NarrowTy}, 1193 {SrcRegs[i], SizeInBits % NarrowTy.getScalarSizeInBits()}) 1194 .getReg(0)); 1195 PartialExtensionReg = DstRegs.back(); 1196 } 1197 } 1198 1199 // Gather the destination registers into the final destination. 1200 Register DstReg = MI.getOperand(0).getReg(); 1201 MIRBuilder.buildMerge(DstReg, DstRegs); 1202 MI.eraseFromParent(); 1203 return Legalized; 1204 } 1205 case TargetOpcode::G_BSWAP: 1206 case TargetOpcode::G_BITREVERSE: { 1207 if (SizeOp0 % NarrowSize != 0) 1208 return UnableToLegalize; 1209 1210 Observer.changingInstr(MI); 1211 SmallVector<Register, 2> SrcRegs, DstRegs; 1212 unsigned NumParts = SizeOp0 / NarrowSize; 1213 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs); 1214 1215 for (unsigned i = 0; i < NumParts; ++i) { 1216 auto DstPart = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy}, 1217 {SrcRegs[NumParts - 1 - i]}); 1218 DstRegs.push_back(DstPart.getReg(0)); 1219 } 1220 1221 MIRBuilder.buildMerge(MI.getOperand(0), DstRegs); 1222 1223 Observer.changedInstr(MI); 1224 MI.eraseFromParent(); 1225 return Legalized; 1226 } 1227 case TargetOpcode::G_PTR_ADD: 1228 case TargetOpcode::G_PTRMASK: { 1229 if (TypeIdx != 1) 1230 return UnableToLegalize; 1231 Observer.changingInstr(MI); 1232 narrowScalarSrc(MI, NarrowTy, 2); 1233 Observer.changedInstr(MI); 1234 return Legalized; 1235 } 1236 case TargetOpcode::G_FPTOUI: { 1237 if (TypeIdx != 0) 1238 return UnableToLegalize; 1239 Observer.changingInstr(MI); 1240 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT); 1241 Observer.changedInstr(MI); 1242 return Legalized; 1243 } 1244 case TargetOpcode::G_FPTOSI: { 1245 if (TypeIdx != 0) 1246 return UnableToLegalize; 1247 Observer.changingInstr(MI); 1248 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_SEXT); 1249 Observer.changedInstr(MI); 1250 return Legalized; 1251 } 1252 case TargetOpcode::G_FPEXT: 1253 if (TypeIdx != 0) 1254 return UnableToLegalize; 1255 Observer.changingInstr(MI); 1256 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_FPEXT); 1257 Observer.changedInstr(MI); 1258 return Legalized; 1259 } 1260 } 1261 1262 Register LegalizerHelper::coerceToScalar(Register Val) { 1263 LLT Ty = MRI.getType(Val); 1264 if (Ty.isScalar()) 1265 return Val; 1266 1267 const DataLayout &DL = MIRBuilder.getDataLayout(); 1268 LLT NewTy = LLT::scalar(Ty.getSizeInBits()); 1269 if (Ty.isPointer()) { 1270 if (DL.isNonIntegralAddressSpace(Ty.getAddressSpace())) 1271 return Register(); 1272 return MIRBuilder.buildPtrToInt(NewTy, Val).getReg(0); 1273 } 1274 1275 Register NewVal = Val; 1276 1277 assert(Ty.isVector()); 1278 LLT EltTy = Ty.getElementType(); 1279 if (EltTy.isPointer()) 1280 NewVal = MIRBuilder.buildPtrToInt(NewTy, NewVal).getReg(0); 1281 return MIRBuilder.buildBitcast(NewTy, NewVal).getReg(0); 1282 } 1283 1284 void LegalizerHelper::widenScalarSrc(MachineInstr &MI, LLT WideTy, 1285 unsigned OpIdx, unsigned ExtOpcode) { 1286 MachineOperand &MO = MI.getOperand(OpIdx); 1287 auto ExtB = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MO}); 1288 MO.setReg(ExtB.getReg(0)); 1289 } 1290 1291 void LegalizerHelper::narrowScalarSrc(MachineInstr &MI, LLT NarrowTy, 1292 unsigned OpIdx) { 1293 MachineOperand &MO = MI.getOperand(OpIdx); 1294 auto ExtB = MIRBuilder.buildTrunc(NarrowTy, MO); 1295 MO.setReg(ExtB.getReg(0)); 1296 } 1297 1298 void LegalizerHelper::widenScalarDst(MachineInstr &MI, LLT WideTy, 1299 unsigned OpIdx, unsigned TruncOpcode) { 1300 MachineOperand &MO = MI.getOperand(OpIdx); 1301 Register DstExt = MRI.createGenericVirtualRegister(WideTy); 1302 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); 1303 MIRBuilder.buildInstr(TruncOpcode, {MO}, {DstExt}); 1304 MO.setReg(DstExt); 1305 } 1306 1307 void LegalizerHelper::narrowScalarDst(MachineInstr &MI, LLT NarrowTy, 1308 unsigned OpIdx, unsigned ExtOpcode) { 1309 MachineOperand &MO = MI.getOperand(OpIdx); 1310 Register DstTrunc = MRI.createGenericVirtualRegister(NarrowTy); 1311 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); 1312 MIRBuilder.buildInstr(ExtOpcode, {MO}, {DstTrunc}); 1313 MO.setReg(DstTrunc); 1314 } 1315 1316 void LegalizerHelper::moreElementsVectorDst(MachineInstr &MI, LLT WideTy, 1317 unsigned OpIdx) { 1318 MachineOperand &MO = MI.getOperand(OpIdx); 1319 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); 1320 MO.setReg(widenWithUnmerge(WideTy, MO.getReg())); 1321 } 1322 1323 void LegalizerHelper::moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy, 1324 unsigned OpIdx) { 1325 MachineOperand &MO = MI.getOperand(OpIdx); 1326 1327 LLT OldTy = MRI.getType(MO.getReg()); 1328 unsigned OldElts = OldTy.getNumElements(); 1329 unsigned NewElts = MoreTy.getNumElements(); 1330 1331 unsigned NumParts = NewElts / OldElts; 1332 1333 // Use concat_vectors if the result is a multiple of the number of elements. 1334 if (NumParts * OldElts == NewElts) { 1335 SmallVector<Register, 8> Parts; 1336 Parts.push_back(MO.getReg()); 1337 1338 Register ImpDef = MIRBuilder.buildUndef(OldTy).getReg(0); 1339 for (unsigned I = 1; I != NumParts; ++I) 1340 Parts.push_back(ImpDef); 1341 1342 auto Concat = MIRBuilder.buildConcatVectors(MoreTy, Parts); 1343 MO.setReg(Concat.getReg(0)); 1344 return; 1345 } 1346 1347 Register MoreReg = MRI.createGenericVirtualRegister(MoreTy); 1348 Register ImpDef = MIRBuilder.buildUndef(MoreTy).getReg(0); 1349 MIRBuilder.buildInsert(MoreReg, ImpDef, MO.getReg(), 0); 1350 MO.setReg(MoreReg); 1351 } 1352 1353 void LegalizerHelper::bitcastSrc(MachineInstr &MI, LLT CastTy, unsigned OpIdx) { 1354 MachineOperand &Op = MI.getOperand(OpIdx); 1355 Op.setReg(MIRBuilder.buildBitcast(CastTy, Op).getReg(0)); 1356 } 1357 1358 void LegalizerHelper::bitcastDst(MachineInstr &MI, LLT CastTy, unsigned OpIdx) { 1359 MachineOperand &MO = MI.getOperand(OpIdx); 1360 Register CastDst = MRI.createGenericVirtualRegister(CastTy); 1361 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); 1362 MIRBuilder.buildBitcast(MO, CastDst); 1363 MO.setReg(CastDst); 1364 } 1365 1366 LegalizerHelper::LegalizeResult 1367 LegalizerHelper::widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx, 1368 LLT WideTy) { 1369 if (TypeIdx != 1) 1370 return UnableToLegalize; 1371 1372 Register DstReg = MI.getOperand(0).getReg(); 1373 LLT DstTy = MRI.getType(DstReg); 1374 if (DstTy.isVector()) 1375 return UnableToLegalize; 1376 1377 Register Src1 = MI.getOperand(1).getReg(); 1378 LLT SrcTy = MRI.getType(Src1); 1379 const int DstSize = DstTy.getSizeInBits(); 1380 const int SrcSize = SrcTy.getSizeInBits(); 1381 const int WideSize = WideTy.getSizeInBits(); 1382 const int NumMerge = (DstSize + WideSize - 1) / WideSize; 1383 1384 unsigned NumOps = MI.getNumOperands(); 1385 unsigned NumSrc = MI.getNumOperands() - 1; 1386 unsigned PartSize = DstTy.getSizeInBits() / NumSrc; 1387 1388 if (WideSize >= DstSize) { 1389 // Directly pack the bits in the target type. 1390 Register ResultReg = MIRBuilder.buildZExt(WideTy, Src1).getReg(0); 1391 1392 for (unsigned I = 2; I != NumOps; ++I) { 1393 const unsigned Offset = (I - 1) * PartSize; 1394 1395 Register SrcReg = MI.getOperand(I).getReg(); 1396 assert(MRI.getType(SrcReg) == LLT::scalar(PartSize)); 1397 1398 auto ZextInput = MIRBuilder.buildZExt(WideTy, SrcReg); 1399 1400 Register NextResult = I + 1 == NumOps && WideTy == DstTy ? DstReg : 1401 MRI.createGenericVirtualRegister(WideTy); 1402 1403 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, Offset); 1404 auto Shl = MIRBuilder.buildShl(WideTy, ZextInput, ShiftAmt); 1405 MIRBuilder.buildOr(NextResult, ResultReg, Shl); 1406 ResultReg = NextResult; 1407 } 1408 1409 if (WideSize > DstSize) 1410 MIRBuilder.buildTrunc(DstReg, ResultReg); 1411 else if (DstTy.isPointer()) 1412 MIRBuilder.buildIntToPtr(DstReg, ResultReg); 1413 1414 MI.eraseFromParent(); 1415 return Legalized; 1416 } 1417 1418 // Unmerge the original values to the GCD type, and recombine to the next 1419 // multiple greater than the original type. 1420 // 1421 // %3:_(s12) = G_MERGE_VALUES %0:_(s4), %1:_(s4), %2:_(s4) -> s6 1422 // %4:_(s2), %5:_(s2) = G_UNMERGE_VALUES %0 1423 // %6:_(s2), %7:_(s2) = G_UNMERGE_VALUES %1 1424 // %8:_(s2), %9:_(s2) = G_UNMERGE_VALUES %2 1425 // %10:_(s6) = G_MERGE_VALUES %4, %5, %6 1426 // %11:_(s6) = G_MERGE_VALUES %7, %8, %9 1427 // %12:_(s12) = G_MERGE_VALUES %10, %11 1428 // 1429 // Padding with undef if necessary: 1430 // 1431 // %2:_(s8) = G_MERGE_VALUES %0:_(s4), %1:_(s4) -> s6 1432 // %3:_(s2), %4:_(s2) = G_UNMERGE_VALUES %0 1433 // %5:_(s2), %6:_(s2) = G_UNMERGE_VALUES %1 1434 // %7:_(s2) = G_IMPLICIT_DEF 1435 // %8:_(s6) = G_MERGE_VALUES %3, %4, %5 1436 // %9:_(s6) = G_MERGE_VALUES %6, %7, %7 1437 // %10:_(s12) = G_MERGE_VALUES %8, %9 1438 1439 const int GCD = greatestCommonDivisor(SrcSize, WideSize); 1440 LLT GCDTy = LLT::scalar(GCD); 1441 1442 SmallVector<Register, 8> Parts; 1443 SmallVector<Register, 8> NewMergeRegs; 1444 SmallVector<Register, 8> Unmerges; 1445 LLT WideDstTy = LLT::scalar(NumMerge * WideSize); 1446 1447 // Decompose the original operands if they don't evenly divide. 1448 for (int I = 1, E = MI.getNumOperands(); I != E; ++I) { 1449 Register SrcReg = MI.getOperand(I).getReg(); 1450 if (GCD == SrcSize) { 1451 Unmerges.push_back(SrcReg); 1452 } else { 1453 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg); 1454 for (int J = 0, JE = Unmerge->getNumOperands() - 1; J != JE; ++J) 1455 Unmerges.push_back(Unmerge.getReg(J)); 1456 } 1457 } 1458 1459 // Pad with undef to the next size that is a multiple of the requested size. 1460 if (static_cast<int>(Unmerges.size()) != NumMerge * WideSize) { 1461 Register UndefReg = MIRBuilder.buildUndef(GCDTy).getReg(0); 1462 for (int I = Unmerges.size(); I != NumMerge * WideSize; ++I) 1463 Unmerges.push_back(UndefReg); 1464 } 1465 1466 const int PartsPerGCD = WideSize / GCD; 1467 1468 // Build merges of each piece. 1469 ArrayRef<Register> Slicer(Unmerges); 1470 for (int I = 0; I != NumMerge; ++I, Slicer = Slicer.drop_front(PartsPerGCD)) { 1471 auto Merge = MIRBuilder.buildMerge(WideTy, Slicer.take_front(PartsPerGCD)); 1472 NewMergeRegs.push_back(Merge.getReg(0)); 1473 } 1474 1475 // A truncate may be necessary if the requested type doesn't evenly divide the 1476 // original result type. 1477 if (DstTy.getSizeInBits() == WideDstTy.getSizeInBits()) { 1478 MIRBuilder.buildMerge(DstReg, NewMergeRegs); 1479 } else { 1480 auto FinalMerge = MIRBuilder.buildMerge(WideDstTy, NewMergeRegs); 1481 MIRBuilder.buildTrunc(DstReg, FinalMerge.getReg(0)); 1482 } 1483 1484 MI.eraseFromParent(); 1485 return Legalized; 1486 } 1487 1488 Register LegalizerHelper::widenWithUnmerge(LLT WideTy, Register OrigReg) { 1489 Register WideReg = MRI.createGenericVirtualRegister(WideTy); 1490 LLT OrigTy = MRI.getType(OrigReg); 1491 LLT LCMTy = getLCMType(WideTy, OrigTy); 1492 1493 const int NumMergeParts = LCMTy.getSizeInBits() / WideTy.getSizeInBits(); 1494 const int NumUnmergeParts = LCMTy.getSizeInBits() / OrigTy.getSizeInBits(); 1495 1496 Register UnmergeSrc = WideReg; 1497 1498 // Create a merge to the LCM type, padding with undef 1499 // %0:_(<3 x s32>) = G_FOO => <4 x s32> 1500 // => 1501 // %1:_(<4 x s32>) = G_FOO 1502 // %2:_(<4 x s32>) = G_IMPLICIT_DEF 1503 // %3:_(<12 x s32>) = G_CONCAT_VECTORS %1, %2, %2 1504 // %0:_(<3 x s32>), %4:_, %5:_, %6:_ = G_UNMERGE_VALUES %3 1505 if (NumMergeParts > 1) { 1506 Register Undef = MIRBuilder.buildUndef(WideTy).getReg(0); 1507 SmallVector<Register, 8> MergeParts(NumMergeParts, Undef); 1508 MergeParts[0] = WideReg; 1509 UnmergeSrc = MIRBuilder.buildMerge(LCMTy, MergeParts).getReg(0); 1510 } 1511 1512 // Unmerge to the original register and pad with dead defs. 1513 SmallVector<Register, 8> UnmergeResults(NumUnmergeParts); 1514 UnmergeResults[0] = OrigReg; 1515 for (int I = 1; I != NumUnmergeParts; ++I) 1516 UnmergeResults[I] = MRI.createGenericVirtualRegister(OrigTy); 1517 1518 MIRBuilder.buildUnmerge(UnmergeResults, UnmergeSrc); 1519 return WideReg; 1520 } 1521 1522 LegalizerHelper::LegalizeResult 1523 LegalizerHelper::widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx, 1524 LLT WideTy) { 1525 if (TypeIdx != 0) 1526 return UnableToLegalize; 1527 1528 int NumDst = MI.getNumOperands() - 1; 1529 Register SrcReg = MI.getOperand(NumDst).getReg(); 1530 LLT SrcTy = MRI.getType(SrcReg); 1531 if (SrcTy.isVector()) 1532 return UnableToLegalize; 1533 1534 Register Dst0Reg = MI.getOperand(0).getReg(); 1535 LLT DstTy = MRI.getType(Dst0Reg); 1536 if (!DstTy.isScalar()) 1537 return UnableToLegalize; 1538 1539 if (WideTy.getSizeInBits() >= SrcTy.getSizeInBits()) { 1540 if (SrcTy.isPointer()) { 1541 const DataLayout &DL = MIRBuilder.getDataLayout(); 1542 if (DL.isNonIntegralAddressSpace(SrcTy.getAddressSpace())) { 1543 LLVM_DEBUG( 1544 dbgs() << "Not casting non-integral address space integer\n"); 1545 return UnableToLegalize; 1546 } 1547 1548 SrcTy = LLT::scalar(SrcTy.getSizeInBits()); 1549 SrcReg = MIRBuilder.buildPtrToInt(SrcTy, SrcReg).getReg(0); 1550 } 1551 1552 // Widen SrcTy to WideTy. This does not affect the result, but since the 1553 // user requested this size, it is probably better handled than SrcTy and 1554 // should reduce the total number of legalization artifacts 1555 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) { 1556 SrcTy = WideTy; 1557 SrcReg = MIRBuilder.buildAnyExt(WideTy, SrcReg).getReg(0); 1558 } 1559 1560 // Theres no unmerge type to target. Directly extract the bits from the 1561 // source type 1562 unsigned DstSize = DstTy.getSizeInBits(); 1563 1564 MIRBuilder.buildTrunc(Dst0Reg, SrcReg); 1565 for (int I = 1; I != NumDst; ++I) { 1566 auto ShiftAmt = MIRBuilder.buildConstant(SrcTy, DstSize * I); 1567 auto Shr = MIRBuilder.buildLShr(SrcTy, SrcReg, ShiftAmt); 1568 MIRBuilder.buildTrunc(MI.getOperand(I), Shr); 1569 } 1570 1571 MI.eraseFromParent(); 1572 return Legalized; 1573 } 1574 1575 // Extend the source to a wider type. 1576 LLT LCMTy = getLCMType(SrcTy, WideTy); 1577 1578 Register WideSrc = SrcReg; 1579 if (LCMTy.getSizeInBits() != SrcTy.getSizeInBits()) { 1580 // TODO: If this is an integral address space, cast to integer and anyext. 1581 if (SrcTy.isPointer()) { 1582 LLVM_DEBUG(dbgs() << "Widening pointer source types not implemented\n"); 1583 return UnableToLegalize; 1584 } 1585 1586 WideSrc = MIRBuilder.buildAnyExt(LCMTy, WideSrc).getReg(0); 1587 } 1588 1589 auto Unmerge = MIRBuilder.buildUnmerge(WideTy, WideSrc); 1590 1591 // Create a sequence of unmerges to the original results. since we may have 1592 // widened the source, we will need to pad the results with dead defs to cover 1593 // the source register. 1594 // e.g. widen s16 to s32: 1595 // %1:_(s16), %2:_(s16), %3:_(s16) = G_UNMERGE_VALUES %0:_(s48) 1596 // 1597 // => 1598 // %4:_(s64) = G_ANYEXT %0:_(s48) 1599 // %5:_(s32), %6:_(s32) = G_UNMERGE_VALUES %4 ; Requested unmerge 1600 // %1:_(s16), %2:_(s16) = G_UNMERGE_VALUES %5 ; unpack to original regs 1601 // %3:_(s16), dead %7 = G_UNMERGE_VALUES %6 ; original reg + extra dead def 1602 1603 const int NumUnmerge = Unmerge->getNumOperands() - 1; 1604 const int PartsPerUnmerge = WideTy.getSizeInBits() / DstTy.getSizeInBits(); 1605 1606 for (int I = 0; I != NumUnmerge; ++I) { 1607 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES); 1608 1609 for (int J = 0; J != PartsPerUnmerge; ++J) { 1610 int Idx = I * PartsPerUnmerge + J; 1611 if (Idx < NumDst) 1612 MIB.addDef(MI.getOperand(Idx).getReg()); 1613 else { 1614 // Create dead def for excess components. 1615 MIB.addDef(MRI.createGenericVirtualRegister(DstTy)); 1616 } 1617 } 1618 1619 MIB.addUse(Unmerge.getReg(I)); 1620 } 1621 1622 MI.eraseFromParent(); 1623 return Legalized; 1624 } 1625 1626 LegalizerHelper::LegalizeResult 1627 LegalizerHelper::widenScalarExtract(MachineInstr &MI, unsigned TypeIdx, 1628 LLT WideTy) { 1629 Register DstReg = MI.getOperand(0).getReg(); 1630 Register SrcReg = MI.getOperand(1).getReg(); 1631 LLT SrcTy = MRI.getType(SrcReg); 1632 1633 LLT DstTy = MRI.getType(DstReg); 1634 unsigned Offset = MI.getOperand(2).getImm(); 1635 1636 if (TypeIdx == 0) { 1637 if (SrcTy.isVector() || DstTy.isVector()) 1638 return UnableToLegalize; 1639 1640 SrcOp Src(SrcReg); 1641 if (SrcTy.isPointer()) { 1642 // Extracts from pointers can be handled only if they are really just 1643 // simple integers. 1644 const DataLayout &DL = MIRBuilder.getDataLayout(); 1645 if (DL.isNonIntegralAddressSpace(SrcTy.getAddressSpace())) 1646 return UnableToLegalize; 1647 1648 LLT SrcAsIntTy = LLT::scalar(SrcTy.getSizeInBits()); 1649 Src = MIRBuilder.buildPtrToInt(SrcAsIntTy, Src); 1650 SrcTy = SrcAsIntTy; 1651 } 1652 1653 if (DstTy.isPointer()) 1654 return UnableToLegalize; 1655 1656 if (Offset == 0) { 1657 // Avoid a shift in the degenerate case. 1658 MIRBuilder.buildTrunc(DstReg, 1659 MIRBuilder.buildAnyExtOrTrunc(WideTy, Src)); 1660 MI.eraseFromParent(); 1661 return Legalized; 1662 } 1663 1664 // Do a shift in the source type. 1665 LLT ShiftTy = SrcTy; 1666 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) { 1667 Src = MIRBuilder.buildAnyExt(WideTy, Src); 1668 ShiftTy = WideTy; 1669 } 1670 1671 auto LShr = MIRBuilder.buildLShr( 1672 ShiftTy, Src, MIRBuilder.buildConstant(ShiftTy, Offset)); 1673 MIRBuilder.buildTrunc(DstReg, LShr); 1674 MI.eraseFromParent(); 1675 return Legalized; 1676 } 1677 1678 if (SrcTy.isScalar()) { 1679 Observer.changingInstr(MI); 1680 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); 1681 Observer.changedInstr(MI); 1682 return Legalized; 1683 } 1684 1685 if (!SrcTy.isVector()) 1686 return UnableToLegalize; 1687 1688 if (DstTy != SrcTy.getElementType()) 1689 return UnableToLegalize; 1690 1691 if (Offset % SrcTy.getScalarSizeInBits() != 0) 1692 return UnableToLegalize; 1693 1694 Observer.changingInstr(MI); 1695 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); 1696 1697 MI.getOperand(2).setImm((WideTy.getSizeInBits() / SrcTy.getSizeInBits()) * 1698 Offset); 1699 widenScalarDst(MI, WideTy.getScalarType(), 0); 1700 Observer.changedInstr(MI); 1701 return Legalized; 1702 } 1703 1704 LegalizerHelper::LegalizeResult 1705 LegalizerHelper::widenScalarInsert(MachineInstr &MI, unsigned TypeIdx, 1706 LLT WideTy) { 1707 if (TypeIdx != 0 || WideTy.isVector()) 1708 return UnableToLegalize; 1709 Observer.changingInstr(MI); 1710 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); 1711 widenScalarDst(MI, WideTy); 1712 Observer.changedInstr(MI); 1713 return Legalized; 1714 } 1715 1716 LegalizerHelper::LegalizeResult 1717 LegalizerHelper::widenScalarAddSubSat(MachineInstr &MI, unsigned TypeIdx, 1718 LLT WideTy) { 1719 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SADDSAT || 1720 MI.getOpcode() == TargetOpcode::G_SSUBSAT; 1721 // We can convert this to: 1722 // 1. Any extend iN to iM 1723 // 2. SHL by M-N 1724 // 3. [US][ADD|SUB]SAT 1725 // 4. L/ASHR by M-N 1726 // 1727 // It may be more efficient to lower this to a min and a max operation in 1728 // the higher precision arithmetic if the promoted operation isn't legal, 1729 // but this decision is up to the target's lowering request. 1730 Register DstReg = MI.getOperand(0).getReg(); 1731 1732 unsigned NewBits = WideTy.getScalarSizeInBits(); 1733 unsigned SHLAmount = NewBits - MRI.getType(DstReg).getScalarSizeInBits(); 1734 1735 auto LHS = MIRBuilder.buildAnyExt(WideTy, MI.getOperand(1)); 1736 auto RHS = MIRBuilder.buildAnyExt(WideTy, MI.getOperand(2)); 1737 auto ShiftK = MIRBuilder.buildConstant(WideTy, SHLAmount); 1738 auto ShiftL = MIRBuilder.buildShl(WideTy, LHS, ShiftK); 1739 auto ShiftR = MIRBuilder.buildShl(WideTy, RHS, ShiftK); 1740 1741 auto WideInst = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy}, 1742 {ShiftL, ShiftR}, MI.getFlags()); 1743 1744 // Use a shift that will preserve the number of sign bits when the trunc is 1745 // folded away. 1746 auto Result = IsSigned ? MIRBuilder.buildAShr(WideTy, WideInst, ShiftK) 1747 : MIRBuilder.buildLShr(WideTy, WideInst, ShiftK); 1748 1749 MIRBuilder.buildTrunc(DstReg, Result); 1750 MI.eraseFromParent(); 1751 return Legalized; 1752 } 1753 1754 LegalizerHelper::LegalizeResult 1755 LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { 1756 switch (MI.getOpcode()) { 1757 default: 1758 return UnableToLegalize; 1759 case TargetOpcode::G_EXTRACT: 1760 return widenScalarExtract(MI, TypeIdx, WideTy); 1761 case TargetOpcode::G_INSERT: 1762 return widenScalarInsert(MI, TypeIdx, WideTy); 1763 case TargetOpcode::G_MERGE_VALUES: 1764 return widenScalarMergeValues(MI, TypeIdx, WideTy); 1765 case TargetOpcode::G_UNMERGE_VALUES: 1766 return widenScalarUnmergeValues(MI, TypeIdx, WideTy); 1767 case TargetOpcode::G_UADDO: 1768 case TargetOpcode::G_USUBO: { 1769 if (TypeIdx == 1) 1770 return UnableToLegalize; // TODO 1771 auto LHSZext = MIRBuilder.buildZExt(WideTy, MI.getOperand(2)); 1772 auto RHSZext = MIRBuilder.buildZExt(WideTy, MI.getOperand(3)); 1773 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_UADDO 1774 ? TargetOpcode::G_ADD 1775 : TargetOpcode::G_SUB; 1776 // Do the arithmetic in the larger type. 1777 auto NewOp = MIRBuilder.buildInstr(Opcode, {WideTy}, {LHSZext, RHSZext}); 1778 LLT OrigTy = MRI.getType(MI.getOperand(0).getReg()); 1779 APInt Mask = 1780 APInt::getLowBitsSet(WideTy.getSizeInBits(), OrigTy.getSizeInBits()); 1781 auto AndOp = MIRBuilder.buildAnd( 1782 WideTy, NewOp, MIRBuilder.buildConstant(WideTy, Mask)); 1783 // There is no overflow if the AndOp is the same as NewOp. 1784 MIRBuilder.buildICmp(CmpInst::ICMP_NE, MI.getOperand(1), NewOp, AndOp); 1785 // Now trunc the NewOp to the original result. 1786 MIRBuilder.buildTrunc(MI.getOperand(0), NewOp); 1787 MI.eraseFromParent(); 1788 return Legalized; 1789 } 1790 case TargetOpcode::G_SADDSAT: 1791 case TargetOpcode::G_SSUBSAT: 1792 case TargetOpcode::G_UADDSAT: 1793 case TargetOpcode::G_USUBSAT: 1794 return widenScalarAddSubSat(MI, TypeIdx, WideTy); 1795 case TargetOpcode::G_CTTZ: 1796 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 1797 case TargetOpcode::G_CTLZ: 1798 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 1799 case TargetOpcode::G_CTPOP: { 1800 if (TypeIdx == 0) { 1801 Observer.changingInstr(MI); 1802 widenScalarDst(MI, WideTy, 0); 1803 Observer.changedInstr(MI); 1804 return Legalized; 1805 } 1806 1807 Register SrcReg = MI.getOperand(1).getReg(); 1808 1809 // First ZEXT the input. 1810 auto MIBSrc = MIRBuilder.buildZExt(WideTy, SrcReg); 1811 LLT CurTy = MRI.getType(SrcReg); 1812 if (MI.getOpcode() == TargetOpcode::G_CTTZ) { 1813 // The count is the same in the larger type except if the original 1814 // value was zero. This can be handled by setting the bit just off 1815 // the top of the original type. 1816 auto TopBit = 1817 APInt::getOneBitSet(WideTy.getSizeInBits(), CurTy.getSizeInBits()); 1818 MIBSrc = MIRBuilder.buildOr( 1819 WideTy, MIBSrc, MIRBuilder.buildConstant(WideTy, TopBit)); 1820 } 1821 1822 // Perform the operation at the larger size. 1823 auto MIBNewOp = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy}, {MIBSrc}); 1824 // This is already the correct result for CTPOP and CTTZs 1825 if (MI.getOpcode() == TargetOpcode::G_CTLZ || 1826 MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF) { 1827 // The correct result is NewOp - (Difference in widety and current ty). 1828 unsigned SizeDiff = WideTy.getSizeInBits() - CurTy.getSizeInBits(); 1829 MIBNewOp = MIRBuilder.buildSub( 1830 WideTy, MIBNewOp, MIRBuilder.buildConstant(WideTy, SizeDiff)); 1831 } 1832 1833 MIRBuilder.buildZExtOrTrunc(MI.getOperand(0), MIBNewOp); 1834 MI.eraseFromParent(); 1835 return Legalized; 1836 } 1837 case TargetOpcode::G_BSWAP: { 1838 Observer.changingInstr(MI); 1839 Register DstReg = MI.getOperand(0).getReg(); 1840 1841 Register ShrReg = MRI.createGenericVirtualRegister(WideTy); 1842 Register DstExt = MRI.createGenericVirtualRegister(WideTy); 1843 Register ShiftAmtReg = MRI.createGenericVirtualRegister(WideTy); 1844 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); 1845 1846 MI.getOperand(0).setReg(DstExt); 1847 1848 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); 1849 1850 LLT Ty = MRI.getType(DstReg); 1851 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits(); 1852 MIRBuilder.buildConstant(ShiftAmtReg, DiffBits); 1853 MIRBuilder.buildLShr(ShrReg, DstExt, ShiftAmtReg); 1854 1855 MIRBuilder.buildTrunc(DstReg, ShrReg); 1856 Observer.changedInstr(MI); 1857 return Legalized; 1858 } 1859 case TargetOpcode::G_BITREVERSE: { 1860 Observer.changingInstr(MI); 1861 1862 Register DstReg = MI.getOperand(0).getReg(); 1863 LLT Ty = MRI.getType(DstReg); 1864 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits(); 1865 1866 Register DstExt = MRI.createGenericVirtualRegister(WideTy); 1867 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); 1868 MI.getOperand(0).setReg(DstExt); 1869 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); 1870 1871 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, DiffBits); 1872 auto Shift = MIRBuilder.buildLShr(WideTy, DstExt, ShiftAmt); 1873 MIRBuilder.buildTrunc(DstReg, Shift); 1874 Observer.changedInstr(MI); 1875 return Legalized; 1876 } 1877 case TargetOpcode::G_FREEZE: 1878 Observer.changingInstr(MI); 1879 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); 1880 widenScalarDst(MI, WideTy); 1881 Observer.changedInstr(MI); 1882 return Legalized; 1883 1884 case TargetOpcode::G_ADD: 1885 case TargetOpcode::G_AND: 1886 case TargetOpcode::G_MUL: 1887 case TargetOpcode::G_OR: 1888 case TargetOpcode::G_XOR: 1889 case TargetOpcode::G_SUB: 1890 // Perform operation at larger width (any extension is fines here, high bits 1891 // don't affect the result) and then truncate the result back to the 1892 // original type. 1893 Observer.changingInstr(MI); 1894 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); 1895 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT); 1896 widenScalarDst(MI, WideTy); 1897 Observer.changedInstr(MI); 1898 return Legalized; 1899 1900 case TargetOpcode::G_SHL: 1901 Observer.changingInstr(MI); 1902 1903 if (TypeIdx == 0) { 1904 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); 1905 widenScalarDst(MI, WideTy); 1906 } else { 1907 assert(TypeIdx == 1); 1908 // The "number of bits to shift" operand must preserve its value as an 1909 // unsigned integer: 1910 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); 1911 } 1912 1913 Observer.changedInstr(MI); 1914 return Legalized; 1915 1916 case TargetOpcode::G_SDIV: 1917 case TargetOpcode::G_SREM: 1918 case TargetOpcode::G_SMIN: 1919 case TargetOpcode::G_SMAX: 1920 Observer.changingInstr(MI); 1921 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT); 1922 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT); 1923 widenScalarDst(MI, WideTy); 1924 Observer.changedInstr(MI); 1925 return Legalized; 1926 1927 case TargetOpcode::G_ASHR: 1928 case TargetOpcode::G_LSHR: 1929 Observer.changingInstr(MI); 1930 1931 if (TypeIdx == 0) { 1932 unsigned CvtOp = MI.getOpcode() == TargetOpcode::G_ASHR ? 1933 TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT; 1934 1935 widenScalarSrc(MI, WideTy, 1, CvtOp); 1936 widenScalarDst(MI, WideTy); 1937 } else { 1938 assert(TypeIdx == 1); 1939 // The "number of bits to shift" operand must preserve its value as an 1940 // unsigned integer: 1941 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); 1942 } 1943 1944 Observer.changedInstr(MI); 1945 return Legalized; 1946 case TargetOpcode::G_UDIV: 1947 case TargetOpcode::G_UREM: 1948 case TargetOpcode::G_UMIN: 1949 case TargetOpcode::G_UMAX: 1950 Observer.changingInstr(MI); 1951 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT); 1952 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); 1953 widenScalarDst(MI, WideTy); 1954 Observer.changedInstr(MI); 1955 return Legalized; 1956 1957 case TargetOpcode::G_SELECT: 1958 Observer.changingInstr(MI); 1959 if (TypeIdx == 0) { 1960 // Perform operation at larger width (any extension is fine here, high 1961 // bits don't affect the result) and then truncate the result back to the 1962 // original type. 1963 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT); 1964 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT); 1965 widenScalarDst(MI, WideTy); 1966 } else { 1967 bool IsVec = MRI.getType(MI.getOperand(1).getReg()).isVector(); 1968 // Explicit extension is required here since high bits affect the result. 1969 widenScalarSrc(MI, WideTy, 1, MIRBuilder.getBoolExtOp(IsVec, false)); 1970 } 1971 Observer.changedInstr(MI); 1972 return Legalized; 1973 1974 case TargetOpcode::G_FPTOSI: 1975 case TargetOpcode::G_FPTOUI: 1976 Observer.changingInstr(MI); 1977 1978 if (TypeIdx == 0) 1979 widenScalarDst(MI, WideTy); 1980 else 1981 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT); 1982 1983 Observer.changedInstr(MI); 1984 return Legalized; 1985 case TargetOpcode::G_SITOFP: 1986 Observer.changingInstr(MI); 1987 1988 if (TypeIdx == 0) 1989 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC); 1990 else 1991 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT); 1992 1993 Observer.changedInstr(MI); 1994 return Legalized; 1995 case TargetOpcode::G_UITOFP: 1996 Observer.changingInstr(MI); 1997 1998 if (TypeIdx == 0) 1999 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC); 2000 else 2001 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT); 2002 2003 Observer.changedInstr(MI); 2004 return Legalized; 2005 case TargetOpcode::G_LOAD: 2006 case TargetOpcode::G_SEXTLOAD: 2007 case TargetOpcode::G_ZEXTLOAD: 2008 Observer.changingInstr(MI); 2009 widenScalarDst(MI, WideTy); 2010 Observer.changedInstr(MI); 2011 return Legalized; 2012 2013 case TargetOpcode::G_STORE: { 2014 if (TypeIdx != 0) 2015 return UnableToLegalize; 2016 2017 LLT Ty = MRI.getType(MI.getOperand(0).getReg()); 2018 if (!isPowerOf2_32(Ty.getSizeInBits())) 2019 return UnableToLegalize; 2020 2021 Observer.changingInstr(MI); 2022 2023 unsigned ExtType = Ty.getScalarSizeInBits() == 1 ? 2024 TargetOpcode::G_ZEXT : TargetOpcode::G_ANYEXT; 2025 widenScalarSrc(MI, WideTy, 0, ExtType); 2026 2027 Observer.changedInstr(MI); 2028 return Legalized; 2029 } 2030 case TargetOpcode::G_CONSTANT: { 2031 MachineOperand &SrcMO = MI.getOperand(1); 2032 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext(); 2033 unsigned ExtOpc = LI.getExtOpcodeForWideningConstant( 2034 MRI.getType(MI.getOperand(0).getReg())); 2035 assert((ExtOpc == TargetOpcode::G_ZEXT || ExtOpc == TargetOpcode::G_SEXT || 2036 ExtOpc == TargetOpcode::G_ANYEXT) && 2037 "Illegal Extend"); 2038 const APInt &SrcVal = SrcMO.getCImm()->getValue(); 2039 const APInt &Val = (ExtOpc == TargetOpcode::G_SEXT) 2040 ? SrcVal.sext(WideTy.getSizeInBits()) 2041 : SrcVal.zext(WideTy.getSizeInBits()); 2042 Observer.changingInstr(MI); 2043 SrcMO.setCImm(ConstantInt::get(Ctx, Val)); 2044 2045 widenScalarDst(MI, WideTy); 2046 Observer.changedInstr(MI); 2047 return Legalized; 2048 } 2049 case TargetOpcode::G_FCONSTANT: { 2050 MachineOperand &SrcMO = MI.getOperand(1); 2051 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext(); 2052 APFloat Val = SrcMO.getFPImm()->getValueAPF(); 2053 bool LosesInfo; 2054 switch (WideTy.getSizeInBits()) { 2055 case 32: 2056 Val.convert(APFloat::IEEEsingle(), APFloat::rmNearestTiesToEven, 2057 &LosesInfo); 2058 break; 2059 case 64: 2060 Val.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven, 2061 &LosesInfo); 2062 break; 2063 default: 2064 return UnableToLegalize; 2065 } 2066 2067 assert(!LosesInfo && "extend should always be lossless"); 2068 2069 Observer.changingInstr(MI); 2070 SrcMO.setFPImm(ConstantFP::get(Ctx, Val)); 2071 2072 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC); 2073 Observer.changedInstr(MI); 2074 return Legalized; 2075 } 2076 case TargetOpcode::G_IMPLICIT_DEF: { 2077 Observer.changingInstr(MI); 2078 widenScalarDst(MI, WideTy); 2079 Observer.changedInstr(MI); 2080 return Legalized; 2081 } 2082 case TargetOpcode::G_BRCOND: 2083 Observer.changingInstr(MI); 2084 widenScalarSrc(MI, WideTy, 0, MIRBuilder.getBoolExtOp(false, false)); 2085 Observer.changedInstr(MI); 2086 return Legalized; 2087 2088 case TargetOpcode::G_FCMP: 2089 Observer.changingInstr(MI); 2090 if (TypeIdx == 0) 2091 widenScalarDst(MI, WideTy); 2092 else { 2093 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT); 2094 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_FPEXT); 2095 } 2096 Observer.changedInstr(MI); 2097 return Legalized; 2098 2099 case TargetOpcode::G_ICMP: 2100 Observer.changingInstr(MI); 2101 if (TypeIdx == 0) 2102 widenScalarDst(MI, WideTy); 2103 else { 2104 unsigned ExtOpcode = CmpInst::isSigned(static_cast<CmpInst::Predicate>( 2105 MI.getOperand(1).getPredicate())) 2106 ? TargetOpcode::G_SEXT 2107 : TargetOpcode::G_ZEXT; 2108 widenScalarSrc(MI, WideTy, 2, ExtOpcode); 2109 widenScalarSrc(MI, WideTy, 3, ExtOpcode); 2110 } 2111 Observer.changedInstr(MI); 2112 return Legalized; 2113 2114 case TargetOpcode::G_PTR_ADD: 2115 assert(TypeIdx == 1 && "unable to legalize pointer of G_PTR_ADD"); 2116 Observer.changingInstr(MI); 2117 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT); 2118 Observer.changedInstr(MI); 2119 return Legalized; 2120 2121 case TargetOpcode::G_PHI: { 2122 assert(TypeIdx == 0 && "Expecting only Idx 0"); 2123 2124 Observer.changingInstr(MI); 2125 for (unsigned I = 1; I < MI.getNumOperands(); I += 2) { 2126 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB(); 2127 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator()); 2128 widenScalarSrc(MI, WideTy, I, TargetOpcode::G_ANYEXT); 2129 } 2130 2131 MachineBasicBlock &MBB = *MI.getParent(); 2132 MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI()); 2133 widenScalarDst(MI, WideTy); 2134 Observer.changedInstr(MI); 2135 return Legalized; 2136 } 2137 case TargetOpcode::G_EXTRACT_VECTOR_ELT: { 2138 if (TypeIdx == 0) { 2139 Register VecReg = MI.getOperand(1).getReg(); 2140 LLT VecTy = MRI.getType(VecReg); 2141 Observer.changingInstr(MI); 2142 2143 widenScalarSrc(MI, LLT::vector(VecTy.getNumElements(), 2144 WideTy.getSizeInBits()), 2145 1, TargetOpcode::G_SEXT); 2146 2147 widenScalarDst(MI, WideTy, 0); 2148 Observer.changedInstr(MI); 2149 return Legalized; 2150 } 2151 2152 if (TypeIdx != 2) 2153 return UnableToLegalize; 2154 Observer.changingInstr(MI); 2155 // TODO: Probably should be zext 2156 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT); 2157 Observer.changedInstr(MI); 2158 return Legalized; 2159 } 2160 case TargetOpcode::G_INSERT_VECTOR_ELT: { 2161 if (TypeIdx == 1) { 2162 Observer.changingInstr(MI); 2163 2164 Register VecReg = MI.getOperand(1).getReg(); 2165 LLT VecTy = MRI.getType(VecReg); 2166 LLT WideVecTy = LLT::vector(VecTy.getNumElements(), WideTy); 2167 2168 widenScalarSrc(MI, WideVecTy, 1, TargetOpcode::G_ANYEXT); 2169 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT); 2170 widenScalarDst(MI, WideVecTy, 0); 2171 Observer.changedInstr(MI); 2172 return Legalized; 2173 } 2174 2175 if (TypeIdx == 2) { 2176 Observer.changingInstr(MI); 2177 // TODO: Probably should be zext 2178 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_SEXT); 2179 Observer.changedInstr(MI); 2180 return Legalized; 2181 } 2182 2183 return UnableToLegalize; 2184 } 2185 case TargetOpcode::G_FADD: 2186 case TargetOpcode::G_FMUL: 2187 case TargetOpcode::G_FSUB: 2188 case TargetOpcode::G_FMA: 2189 case TargetOpcode::G_FMAD: 2190 case TargetOpcode::G_FNEG: 2191 case TargetOpcode::G_FABS: 2192 case TargetOpcode::G_FCANONICALIZE: 2193 case TargetOpcode::G_FMINNUM: 2194 case TargetOpcode::G_FMAXNUM: 2195 case TargetOpcode::G_FMINNUM_IEEE: 2196 case TargetOpcode::G_FMAXNUM_IEEE: 2197 case TargetOpcode::G_FMINIMUM: 2198 case TargetOpcode::G_FMAXIMUM: 2199 case TargetOpcode::G_FDIV: 2200 case TargetOpcode::G_FREM: 2201 case TargetOpcode::G_FCEIL: 2202 case TargetOpcode::G_FFLOOR: 2203 case TargetOpcode::G_FCOS: 2204 case TargetOpcode::G_FSIN: 2205 case TargetOpcode::G_FLOG10: 2206 case TargetOpcode::G_FLOG: 2207 case TargetOpcode::G_FLOG2: 2208 case TargetOpcode::G_FRINT: 2209 case TargetOpcode::G_FNEARBYINT: 2210 case TargetOpcode::G_FSQRT: 2211 case TargetOpcode::G_FEXP: 2212 case TargetOpcode::G_FEXP2: 2213 case TargetOpcode::G_FPOW: 2214 case TargetOpcode::G_INTRINSIC_TRUNC: 2215 case TargetOpcode::G_INTRINSIC_ROUND: 2216 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: 2217 assert(TypeIdx == 0); 2218 Observer.changingInstr(MI); 2219 2220 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) 2221 widenScalarSrc(MI, WideTy, I, TargetOpcode::G_FPEXT); 2222 2223 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC); 2224 Observer.changedInstr(MI); 2225 return Legalized; 2226 case TargetOpcode::G_FPOWI: { 2227 if (TypeIdx != 0) 2228 return UnableToLegalize; 2229 Observer.changingInstr(MI); 2230 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT); 2231 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC); 2232 Observer.changedInstr(MI); 2233 return Legalized; 2234 } 2235 case TargetOpcode::G_INTTOPTR: 2236 if (TypeIdx != 1) 2237 return UnableToLegalize; 2238 2239 Observer.changingInstr(MI); 2240 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT); 2241 Observer.changedInstr(MI); 2242 return Legalized; 2243 case TargetOpcode::G_PTRTOINT: 2244 if (TypeIdx != 0) 2245 return UnableToLegalize; 2246 2247 Observer.changingInstr(MI); 2248 widenScalarDst(MI, WideTy, 0); 2249 Observer.changedInstr(MI); 2250 return Legalized; 2251 case TargetOpcode::G_BUILD_VECTOR: { 2252 Observer.changingInstr(MI); 2253 2254 const LLT WideEltTy = TypeIdx == 1 ? WideTy : WideTy.getElementType(); 2255 for (int I = 1, E = MI.getNumOperands(); I != E; ++I) 2256 widenScalarSrc(MI, WideEltTy, I, TargetOpcode::G_ANYEXT); 2257 2258 // Avoid changing the result vector type if the source element type was 2259 // requested. 2260 if (TypeIdx == 1) { 2261 MI.setDesc(MIRBuilder.getTII().get(TargetOpcode::G_BUILD_VECTOR_TRUNC)); 2262 } else { 2263 widenScalarDst(MI, WideTy, 0); 2264 } 2265 2266 Observer.changedInstr(MI); 2267 return Legalized; 2268 } 2269 case TargetOpcode::G_SEXT_INREG: 2270 if (TypeIdx != 0) 2271 return UnableToLegalize; 2272 2273 Observer.changingInstr(MI); 2274 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); 2275 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_TRUNC); 2276 Observer.changedInstr(MI); 2277 return Legalized; 2278 case TargetOpcode::G_PTRMASK: { 2279 if (TypeIdx != 1) 2280 return UnableToLegalize; 2281 Observer.changingInstr(MI); 2282 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); 2283 Observer.changedInstr(MI); 2284 return Legalized; 2285 } 2286 } 2287 } 2288 2289 static void getUnmergePieces(SmallVectorImpl<Register> &Pieces, 2290 MachineIRBuilder &B, Register Src, LLT Ty) { 2291 auto Unmerge = B.buildUnmerge(Ty, Src); 2292 for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I) 2293 Pieces.push_back(Unmerge.getReg(I)); 2294 } 2295 2296 LegalizerHelper::LegalizeResult 2297 LegalizerHelper::lowerBitcast(MachineInstr &MI) { 2298 Register Dst = MI.getOperand(0).getReg(); 2299 Register Src = MI.getOperand(1).getReg(); 2300 LLT DstTy = MRI.getType(Dst); 2301 LLT SrcTy = MRI.getType(Src); 2302 2303 if (SrcTy.isVector()) { 2304 LLT SrcEltTy = SrcTy.getElementType(); 2305 SmallVector<Register, 8> SrcRegs; 2306 2307 if (DstTy.isVector()) { 2308 int NumDstElt = DstTy.getNumElements(); 2309 int NumSrcElt = SrcTy.getNumElements(); 2310 2311 LLT DstEltTy = DstTy.getElementType(); 2312 LLT DstCastTy = DstEltTy; // Intermediate bitcast result type 2313 LLT SrcPartTy = SrcEltTy; // Original unmerge result type. 2314 2315 // If there's an element size mismatch, insert intermediate casts to match 2316 // the result element type. 2317 if (NumSrcElt < NumDstElt) { // Source element type is larger. 2318 // %1:_(<4 x s8>) = G_BITCAST %0:_(<2 x s16>) 2319 // 2320 // => 2321 // 2322 // %2:_(s16), %3:_(s16) = G_UNMERGE_VALUES %0 2323 // %3:_(<2 x s8>) = G_BITCAST %2 2324 // %4:_(<2 x s8>) = G_BITCAST %3 2325 // %1:_(<4 x s16>) = G_CONCAT_VECTORS %3, %4 2326 DstCastTy = LLT::vector(NumDstElt / NumSrcElt, DstEltTy); 2327 SrcPartTy = SrcEltTy; 2328 } else if (NumSrcElt > NumDstElt) { // Source element type is smaller. 2329 // 2330 // %1:_(<2 x s16>) = G_BITCAST %0:_(<4 x s8>) 2331 // 2332 // => 2333 // 2334 // %2:_(<2 x s8>), %3:_(<2 x s8>) = G_UNMERGE_VALUES %0 2335 // %3:_(s16) = G_BITCAST %2 2336 // %4:_(s16) = G_BITCAST %3 2337 // %1:_(<2 x s16>) = G_BUILD_VECTOR %3, %4 2338 SrcPartTy = LLT::vector(NumSrcElt / NumDstElt, SrcEltTy); 2339 DstCastTy = DstEltTy; 2340 } 2341 2342 getUnmergePieces(SrcRegs, MIRBuilder, Src, SrcPartTy); 2343 for (Register &SrcReg : SrcRegs) 2344 SrcReg = MIRBuilder.buildBitcast(DstCastTy, SrcReg).getReg(0); 2345 } else 2346 getUnmergePieces(SrcRegs, MIRBuilder, Src, SrcEltTy); 2347 2348 MIRBuilder.buildMerge(Dst, SrcRegs); 2349 MI.eraseFromParent(); 2350 return Legalized; 2351 } 2352 2353 if (DstTy.isVector()) { 2354 SmallVector<Register, 8> SrcRegs; 2355 getUnmergePieces(SrcRegs, MIRBuilder, Src, DstTy.getElementType()); 2356 MIRBuilder.buildMerge(Dst, SrcRegs); 2357 MI.eraseFromParent(); 2358 return Legalized; 2359 } 2360 2361 return UnableToLegalize; 2362 } 2363 2364 /// Perform a G_EXTRACT_VECTOR_ELT in a different sized vector element. If this 2365 /// is casting to a vector with a smaller element size, perform multiple element 2366 /// extracts and merge the results. If this is coercing to a vector with larger 2367 /// elements, index the bitcasted vector and extract the target element with bit 2368 /// operations. This is intended to force the indexing in the native register 2369 /// size for architectures that can dynamically index the register file. 2370 LegalizerHelper::LegalizeResult 2371 LegalizerHelper::bitcastExtractVectorElt(MachineInstr &MI, unsigned TypeIdx, 2372 LLT CastTy) { 2373 if (TypeIdx != 1) 2374 return UnableToLegalize; 2375 2376 Register Dst = MI.getOperand(0).getReg(); 2377 Register SrcVec = MI.getOperand(1).getReg(); 2378 Register Idx = MI.getOperand(2).getReg(); 2379 LLT SrcVecTy = MRI.getType(SrcVec); 2380 LLT IdxTy = MRI.getType(Idx); 2381 2382 LLT SrcEltTy = SrcVecTy.getElementType(); 2383 unsigned NewNumElts = CastTy.isVector() ? CastTy.getNumElements() : 1; 2384 unsigned OldNumElts = SrcVecTy.getNumElements(); 2385 2386 LLT NewEltTy = CastTy.isVector() ? CastTy.getElementType() : CastTy; 2387 Register CastVec = MIRBuilder.buildBitcast(CastTy, SrcVec).getReg(0); 2388 2389 const unsigned NewEltSize = NewEltTy.getSizeInBits(); 2390 const unsigned OldEltSize = SrcEltTy.getSizeInBits(); 2391 if (NewNumElts > OldNumElts) { 2392 // Decreasing the vector element size 2393 // 2394 // e.g. i64 = extract_vector_elt x:v2i64, y:i32 2395 // => 2396 // v4i32:castx = bitcast x:v2i64 2397 // 2398 // i64 = bitcast 2399 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))), 2400 // (i32 (extract_vector_elt castx, (2 * y + 1))) 2401 // 2402 if (NewNumElts % OldNumElts != 0) 2403 return UnableToLegalize; 2404 2405 // Type of the intermediate result vector. 2406 const unsigned NewEltsPerOldElt = NewNumElts / OldNumElts; 2407 LLT MidTy = LLT::scalarOrVector(NewEltsPerOldElt, NewEltTy); 2408 2409 auto NewEltsPerOldEltK = MIRBuilder.buildConstant(IdxTy, NewEltsPerOldElt); 2410 2411 SmallVector<Register, 8> NewOps(NewEltsPerOldElt); 2412 auto NewBaseIdx = MIRBuilder.buildMul(IdxTy, Idx, NewEltsPerOldEltK); 2413 2414 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) { 2415 auto IdxOffset = MIRBuilder.buildConstant(IdxTy, I); 2416 auto TmpIdx = MIRBuilder.buildAdd(IdxTy, NewBaseIdx, IdxOffset); 2417 auto Elt = MIRBuilder.buildExtractVectorElement(NewEltTy, CastVec, TmpIdx); 2418 NewOps[I] = Elt.getReg(0); 2419 } 2420 2421 auto NewVec = MIRBuilder.buildBuildVector(MidTy, NewOps); 2422 MIRBuilder.buildBitcast(Dst, NewVec); 2423 MI.eraseFromParent(); 2424 return Legalized; 2425 } 2426 2427 if (NewNumElts < OldNumElts) { 2428 if (NewEltSize % OldEltSize != 0) 2429 return UnableToLegalize; 2430 2431 // This only depends on powers of 2 because we use bit tricks to figure out 2432 // the bit offset we need to shift to get the target element. A general 2433 // expansion could emit division/multiply. 2434 if (!isPowerOf2_32(NewEltSize / OldEltSize)) 2435 return UnableToLegalize; 2436 2437 // Increasing the vector element size. 2438 // %elt:_(small_elt) = G_EXTRACT_VECTOR_ELT %vec:_(<N x small_elt>), %idx 2439 // 2440 // => 2441 // 2442 // %cast = G_BITCAST %vec 2443 // %scaled_idx = G_LSHR %idx, Log2(DstEltSize / SrcEltSize) 2444 // %wide_elt = G_EXTRACT_VECTOR_ELT %cast, %scaled_idx 2445 // %offset_idx = G_AND %idx, ~(-1 << Log2(DstEltSize / SrcEltSize)) 2446 // %offset_bits = G_SHL %offset_idx, Log2(SrcEltSize) 2447 // %elt_bits = G_LSHR %wide_elt, %offset_bits 2448 // %elt = G_TRUNC %elt_bits 2449 2450 const unsigned Log2EltRatio = Log2_32(NewEltSize / OldEltSize); 2451 auto Log2Ratio = MIRBuilder.buildConstant(IdxTy, Log2EltRatio); 2452 2453 // Divide to get the index in the wider element type. 2454 auto ScaledIdx = MIRBuilder.buildLShr(IdxTy, Idx, Log2Ratio); 2455 2456 Register WideElt = CastVec; 2457 if (CastTy.isVector()) { 2458 WideElt = MIRBuilder.buildExtractVectorElement(NewEltTy, CastVec, 2459 ScaledIdx).getReg(0); 2460 } 2461 2462 // Now figure out the amount we need to shift to get the target bits. 2463 auto OffsetMask = MIRBuilder.buildConstant( 2464 IdxTy, ~(APInt::getAllOnesValue(IdxTy.getSizeInBits()) << Log2EltRatio)); 2465 auto OffsetIdx = MIRBuilder.buildAnd(IdxTy, Idx, OffsetMask); 2466 auto OffsetBits = MIRBuilder.buildShl( 2467 IdxTy, OffsetIdx, 2468 MIRBuilder.buildConstant(IdxTy, Log2_32(OldEltSize))); 2469 2470 // Shift the wide element to get the target element. 2471 auto ExtractedBits = MIRBuilder.buildLShr(NewEltTy, WideElt, OffsetBits); 2472 MIRBuilder.buildTrunc(Dst, ExtractedBits); 2473 MI.eraseFromParent(); 2474 return Legalized; 2475 } 2476 2477 return UnableToLegalize; 2478 } 2479 2480 LegalizerHelper::LegalizeResult 2481 LegalizerHelper::lowerLoad(MachineInstr &MI) { 2482 // Lower to a memory-width G_LOAD and a G_SEXT/G_ZEXT/G_ANYEXT 2483 Register DstReg = MI.getOperand(0).getReg(); 2484 Register PtrReg = MI.getOperand(1).getReg(); 2485 LLT DstTy = MRI.getType(DstReg); 2486 auto &MMO = **MI.memoperands_begin(); 2487 2488 if (DstTy.getSizeInBits() == MMO.getSizeInBits()) { 2489 if (MI.getOpcode() == TargetOpcode::G_LOAD) { 2490 // This load needs splitting into power of 2 sized loads. 2491 if (DstTy.isVector()) 2492 return UnableToLegalize; 2493 if (isPowerOf2_32(DstTy.getSizeInBits())) 2494 return UnableToLegalize; // Don't know what we're being asked to do. 2495 2496 // Our strategy here is to generate anyextending loads for the smaller 2497 // types up to next power-2 result type, and then combine the two larger 2498 // result values together, before truncating back down to the non-pow-2 2499 // type. 2500 // E.g. v1 = i24 load => 2501 // v2 = i32 zextload (2 byte) 2502 // v3 = i32 load (1 byte) 2503 // v4 = i32 shl v3, 16 2504 // v5 = i32 or v4, v2 2505 // v1 = i24 trunc v5 2506 // By doing this we generate the correct truncate which should get 2507 // combined away as an artifact with a matching extend. 2508 uint64_t LargeSplitSize = PowerOf2Floor(DstTy.getSizeInBits()); 2509 uint64_t SmallSplitSize = DstTy.getSizeInBits() - LargeSplitSize; 2510 2511 MachineFunction &MF = MIRBuilder.getMF(); 2512 MachineMemOperand *LargeMMO = 2513 MF.getMachineMemOperand(&MMO, 0, LargeSplitSize / 8); 2514 MachineMemOperand *SmallMMO = MF.getMachineMemOperand( 2515 &MMO, LargeSplitSize / 8, SmallSplitSize / 8); 2516 2517 LLT PtrTy = MRI.getType(PtrReg); 2518 unsigned AnyExtSize = NextPowerOf2(DstTy.getSizeInBits()); 2519 LLT AnyExtTy = LLT::scalar(AnyExtSize); 2520 Register LargeLdReg = MRI.createGenericVirtualRegister(AnyExtTy); 2521 Register SmallLdReg = MRI.createGenericVirtualRegister(AnyExtTy); 2522 auto LargeLoad = MIRBuilder.buildLoadInstr( 2523 TargetOpcode::G_ZEXTLOAD, LargeLdReg, PtrReg, *LargeMMO); 2524 2525 auto OffsetCst = MIRBuilder.buildConstant( 2526 LLT::scalar(PtrTy.getSizeInBits()), LargeSplitSize / 8); 2527 Register PtrAddReg = MRI.createGenericVirtualRegister(PtrTy); 2528 auto SmallPtr = 2529 MIRBuilder.buildPtrAdd(PtrAddReg, PtrReg, OffsetCst.getReg(0)); 2530 auto SmallLoad = MIRBuilder.buildLoad(SmallLdReg, SmallPtr.getReg(0), 2531 *SmallMMO); 2532 2533 auto ShiftAmt = MIRBuilder.buildConstant(AnyExtTy, LargeSplitSize); 2534 auto Shift = MIRBuilder.buildShl(AnyExtTy, SmallLoad, ShiftAmt); 2535 auto Or = MIRBuilder.buildOr(AnyExtTy, Shift, LargeLoad); 2536 MIRBuilder.buildTrunc(DstReg, {Or.getReg(0)}); 2537 MI.eraseFromParent(); 2538 return Legalized; 2539 } 2540 2541 MIRBuilder.buildLoad(DstReg, PtrReg, MMO); 2542 MI.eraseFromParent(); 2543 return Legalized; 2544 } 2545 2546 if (DstTy.isScalar()) { 2547 Register TmpReg = 2548 MRI.createGenericVirtualRegister(LLT::scalar(MMO.getSizeInBits())); 2549 MIRBuilder.buildLoad(TmpReg, PtrReg, MMO); 2550 switch (MI.getOpcode()) { 2551 default: 2552 llvm_unreachable("Unexpected opcode"); 2553 case TargetOpcode::G_LOAD: 2554 MIRBuilder.buildAnyExtOrTrunc(DstReg, TmpReg); 2555 break; 2556 case TargetOpcode::G_SEXTLOAD: 2557 MIRBuilder.buildSExt(DstReg, TmpReg); 2558 break; 2559 case TargetOpcode::G_ZEXTLOAD: 2560 MIRBuilder.buildZExt(DstReg, TmpReg); 2561 break; 2562 } 2563 2564 MI.eraseFromParent(); 2565 return Legalized; 2566 } 2567 2568 return UnableToLegalize; 2569 } 2570 2571 LegalizerHelper::LegalizeResult 2572 LegalizerHelper::lowerStore(MachineInstr &MI) { 2573 // Lower a non-power of 2 store into multiple pow-2 stores. 2574 // E.g. split an i24 store into an i16 store + i8 store. 2575 // We do this by first extending the stored value to the next largest power 2576 // of 2 type, and then using truncating stores to store the components. 2577 // By doing this, likewise with G_LOAD, generate an extend that can be 2578 // artifact-combined away instead of leaving behind extracts. 2579 Register SrcReg = MI.getOperand(0).getReg(); 2580 Register PtrReg = MI.getOperand(1).getReg(); 2581 LLT SrcTy = MRI.getType(SrcReg); 2582 MachineMemOperand &MMO = **MI.memoperands_begin(); 2583 if (SrcTy.getSizeInBits() != MMO.getSizeInBits()) 2584 return UnableToLegalize; 2585 if (SrcTy.isVector()) 2586 return UnableToLegalize; 2587 if (isPowerOf2_32(SrcTy.getSizeInBits())) 2588 return UnableToLegalize; // Don't know what we're being asked to do. 2589 2590 // Extend to the next pow-2. 2591 const LLT ExtendTy = LLT::scalar(NextPowerOf2(SrcTy.getSizeInBits())); 2592 auto ExtVal = MIRBuilder.buildAnyExt(ExtendTy, SrcReg); 2593 2594 // Obtain the smaller value by shifting away the larger value. 2595 uint64_t LargeSplitSize = PowerOf2Floor(SrcTy.getSizeInBits()); 2596 uint64_t SmallSplitSize = SrcTy.getSizeInBits() - LargeSplitSize; 2597 auto ShiftAmt = MIRBuilder.buildConstant(ExtendTy, LargeSplitSize); 2598 auto SmallVal = MIRBuilder.buildLShr(ExtendTy, ExtVal, ShiftAmt); 2599 2600 // Generate the PtrAdd and truncating stores. 2601 LLT PtrTy = MRI.getType(PtrReg); 2602 auto OffsetCst = MIRBuilder.buildConstant( 2603 LLT::scalar(PtrTy.getSizeInBits()), LargeSplitSize / 8); 2604 Register PtrAddReg = MRI.createGenericVirtualRegister(PtrTy); 2605 auto SmallPtr = 2606 MIRBuilder.buildPtrAdd(PtrAddReg, PtrReg, OffsetCst.getReg(0)); 2607 2608 MachineFunction &MF = MIRBuilder.getMF(); 2609 MachineMemOperand *LargeMMO = 2610 MF.getMachineMemOperand(&MMO, 0, LargeSplitSize / 8); 2611 MachineMemOperand *SmallMMO = 2612 MF.getMachineMemOperand(&MMO, LargeSplitSize / 8, SmallSplitSize / 8); 2613 MIRBuilder.buildStore(ExtVal.getReg(0), PtrReg, *LargeMMO); 2614 MIRBuilder.buildStore(SmallVal.getReg(0), SmallPtr.getReg(0), *SmallMMO); 2615 MI.eraseFromParent(); 2616 return Legalized; 2617 } 2618 2619 LegalizerHelper::LegalizeResult 2620 LegalizerHelper::bitcast(MachineInstr &MI, unsigned TypeIdx, LLT CastTy) { 2621 switch (MI.getOpcode()) { 2622 case TargetOpcode::G_LOAD: { 2623 if (TypeIdx != 0) 2624 return UnableToLegalize; 2625 2626 Observer.changingInstr(MI); 2627 bitcastDst(MI, CastTy, 0); 2628 Observer.changedInstr(MI); 2629 return Legalized; 2630 } 2631 case TargetOpcode::G_STORE: { 2632 if (TypeIdx != 0) 2633 return UnableToLegalize; 2634 2635 Observer.changingInstr(MI); 2636 bitcastSrc(MI, CastTy, 0); 2637 Observer.changedInstr(MI); 2638 return Legalized; 2639 } 2640 case TargetOpcode::G_SELECT: { 2641 if (TypeIdx != 0) 2642 return UnableToLegalize; 2643 2644 if (MRI.getType(MI.getOperand(1).getReg()).isVector()) { 2645 LLVM_DEBUG( 2646 dbgs() << "bitcast action not implemented for vector select\n"); 2647 return UnableToLegalize; 2648 } 2649 2650 Observer.changingInstr(MI); 2651 bitcastSrc(MI, CastTy, 2); 2652 bitcastSrc(MI, CastTy, 3); 2653 bitcastDst(MI, CastTy, 0); 2654 Observer.changedInstr(MI); 2655 return Legalized; 2656 } 2657 case TargetOpcode::G_AND: 2658 case TargetOpcode::G_OR: 2659 case TargetOpcode::G_XOR: { 2660 Observer.changingInstr(MI); 2661 bitcastSrc(MI, CastTy, 1); 2662 bitcastSrc(MI, CastTy, 2); 2663 bitcastDst(MI, CastTy, 0); 2664 Observer.changedInstr(MI); 2665 return Legalized; 2666 } 2667 case TargetOpcode::G_EXTRACT_VECTOR_ELT: 2668 return bitcastExtractVectorElt(MI, TypeIdx, CastTy); 2669 default: 2670 return UnableToLegalize; 2671 } 2672 } 2673 2674 // Legalize an instruction by changing the opcode in place. 2675 void LegalizerHelper::changeOpcode(MachineInstr &MI, unsigned NewOpcode) { 2676 Observer.changingInstr(MI); 2677 MI.setDesc(MIRBuilder.getTII().get(NewOpcode)); 2678 Observer.changedInstr(MI); 2679 } 2680 2681 LegalizerHelper::LegalizeResult 2682 LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { 2683 using namespace TargetOpcode; 2684 2685 switch(MI.getOpcode()) { 2686 default: 2687 return UnableToLegalize; 2688 case TargetOpcode::G_BITCAST: 2689 return lowerBitcast(MI); 2690 case TargetOpcode::G_SREM: 2691 case TargetOpcode::G_UREM: { 2692 auto Quot = 2693 MIRBuilder.buildInstr(MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV, {Ty}, 2694 {MI.getOperand(1), MI.getOperand(2)}); 2695 2696 auto Prod = MIRBuilder.buildMul(Ty, Quot, MI.getOperand(2)); 2697 MIRBuilder.buildSub(MI.getOperand(0), MI.getOperand(1), Prod); 2698 MI.eraseFromParent(); 2699 return Legalized; 2700 } 2701 case TargetOpcode::G_SADDO: 2702 case TargetOpcode::G_SSUBO: 2703 return lowerSADDO_SSUBO(MI); 2704 case TargetOpcode::G_SMULO: 2705 case TargetOpcode::G_UMULO: { 2706 // Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the 2707 // result. 2708 Register Res = MI.getOperand(0).getReg(); 2709 Register Overflow = MI.getOperand(1).getReg(); 2710 Register LHS = MI.getOperand(2).getReg(); 2711 Register RHS = MI.getOperand(3).getReg(); 2712 2713 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO 2714 ? TargetOpcode::G_SMULH 2715 : TargetOpcode::G_UMULH; 2716 2717 Observer.changingInstr(MI); 2718 const auto &TII = MIRBuilder.getTII(); 2719 MI.setDesc(TII.get(TargetOpcode::G_MUL)); 2720 MI.RemoveOperand(1); 2721 Observer.changedInstr(MI); 2722 2723 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); 2724 2725 auto HiPart = MIRBuilder.buildInstr(Opcode, {Ty}, {LHS, RHS}); 2726 auto Zero = MIRBuilder.buildConstant(Ty, 0); 2727 2728 // For *signed* multiply, overflow is detected by checking: 2729 // (hi != (lo >> bitwidth-1)) 2730 if (Opcode == TargetOpcode::G_SMULH) { 2731 auto ShiftAmt = MIRBuilder.buildConstant(Ty, Ty.getSizeInBits() - 1); 2732 auto Shifted = MIRBuilder.buildAShr(Ty, Res, ShiftAmt); 2733 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Shifted); 2734 } else { 2735 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Zero); 2736 } 2737 return Legalized; 2738 } 2739 case TargetOpcode::G_FNEG: { 2740 // TODO: Handle vector types once we are able to 2741 // represent them. 2742 if (Ty.isVector()) 2743 return UnableToLegalize; 2744 Register Res = MI.getOperand(0).getReg(); 2745 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext(); 2746 Type *ZeroTy = getFloatTypeForLLT(Ctx, Ty); 2747 if (!ZeroTy) 2748 return UnableToLegalize; 2749 ConstantFP &ZeroForNegation = 2750 *cast<ConstantFP>(ConstantFP::getZeroValueForNegation(ZeroTy)); 2751 auto Zero = MIRBuilder.buildFConstant(Ty, ZeroForNegation); 2752 Register SubByReg = MI.getOperand(1).getReg(); 2753 Register ZeroReg = Zero.getReg(0); 2754 MIRBuilder.buildFSub(Res, ZeroReg, SubByReg, MI.getFlags()); 2755 MI.eraseFromParent(); 2756 return Legalized; 2757 } 2758 case TargetOpcode::G_FSUB: { 2759 // Lower (G_FSUB LHS, RHS) to (G_FADD LHS, (G_FNEG RHS)). 2760 // First, check if G_FNEG is marked as Lower. If so, we may 2761 // end up with an infinite loop as G_FSUB is used to legalize G_FNEG. 2762 if (LI.getAction({G_FNEG, {Ty}}).Action == Lower) 2763 return UnableToLegalize; 2764 Register Res = MI.getOperand(0).getReg(); 2765 Register LHS = MI.getOperand(1).getReg(); 2766 Register RHS = MI.getOperand(2).getReg(); 2767 Register Neg = MRI.createGenericVirtualRegister(Ty); 2768 MIRBuilder.buildFNeg(Neg, RHS); 2769 MIRBuilder.buildFAdd(Res, LHS, Neg, MI.getFlags()); 2770 MI.eraseFromParent(); 2771 return Legalized; 2772 } 2773 case TargetOpcode::G_FMAD: 2774 return lowerFMad(MI); 2775 case TargetOpcode::G_FFLOOR: 2776 return lowerFFloor(MI); 2777 case TargetOpcode::G_INTRINSIC_ROUND: 2778 return lowerIntrinsicRound(MI); 2779 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: { 2780 // Since round even is the assumed rounding mode for unconstrained FP 2781 // operations, rint and roundeven are the same operation. 2782 changeOpcode(MI, TargetOpcode::G_FRINT); 2783 return Legalized; 2784 } 2785 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: { 2786 Register OldValRes = MI.getOperand(0).getReg(); 2787 Register SuccessRes = MI.getOperand(1).getReg(); 2788 Register Addr = MI.getOperand(2).getReg(); 2789 Register CmpVal = MI.getOperand(3).getReg(); 2790 Register NewVal = MI.getOperand(4).getReg(); 2791 MIRBuilder.buildAtomicCmpXchg(OldValRes, Addr, CmpVal, NewVal, 2792 **MI.memoperands_begin()); 2793 MIRBuilder.buildICmp(CmpInst::ICMP_EQ, SuccessRes, OldValRes, CmpVal); 2794 MI.eraseFromParent(); 2795 return Legalized; 2796 } 2797 case TargetOpcode::G_LOAD: 2798 case TargetOpcode::G_SEXTLOAD: 2799 case TargetOpcode::G_ZEXTLOAD: 2800 return lowerLoad(MI); 2801 case TargetOpcode::G_STORE: 2802 return lowerStore(MI); 2803 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 2804 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 2805 case TargetOpcode::G_CTLZ: 2806 case TargetOpcode::G_CTTZ: 2807 case TargetOpcode::G_CTPOP: 2808 return lowerBitCount(MI, TypeIdx, Ty); 2809 case G_UADDO: { 2810 Register Res = MI.getOperand(0).getReg(); 2811 Register CarryOut = MI.getOperand(1).getReg(); 2812 Register LHS = MI.getOperand(2).getReg(); 2813 Register RHS = MI.getOperand(3).getReg(); 2814 2815 MIRBuilder.buildAdd(Res, LHS, RHS); 2816 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, RHS); 2817 2818 MI.eraseFromParent(); 2819 return Legalized; 2820 } 2821 case G_UADDE: { 2822 Register Res = MI.getOperand(0).getReg(); 2823 Register CarryOut = MI.getOperand(1).getReg(); 2824 Register LHS = MI.getOperand(2).getReg(); 2825 Register RHS = MI.getOperand(3).getReg(); 2826 Register CarryIn = MI.getOperand(4).getReg(); 2827 LLT Ty = MRI.getType(Res); 2828 2829 auto TmpRes = MIRBuilder.buildAdd(Ty, LHS, RHS); 2830 auto ZExtCarryIn = MIRBuilder.buildZExt(Ty, CarryIn); 2831 MIRBuilder.buildAdd(Res, TmpRes, ZExtCarryIn); 2832 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, LHS); 2833 2834 MI.eraseFromParent(); 2835 return Legalized; 2836 } 2837 case G_USUBO: { 2838 Register Res = MI.getOperand(0).getReg(); 2839 Register BorrowOut = MI.getOperand(1).getReg(); 2840 Register LHS = MI.getOperand(2).getReg(); 2841 Register RHS = MI.getOperand(3).getReg(); 2842 2843 MIRBuilder.buildSub(Res, LHS, RHS); 2844 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, BorrowOut, LHS, RHS); 2845 2846 MI.eraseFromParent(); 2847 return Legalized; 2848 } 2849 case G_USUBE: { 2850 Register Res = MI.getOperand(0).getReg(); 2851 Register BorrowOut = MI.getOperand(1).getReg(); 2852 Register LHS = MI.getOperand(2).getReg(); 2853 Register RHS = MI.getOperand(3).getReg(); 2854 Register BorrowIn = MI.getOperand(4).getReg(); 2855 const LLT CondTy = MRI.getType(BorrowOut); 2856 const LLT Ty = MRI.getType(Res); 2857 2858 auto TmpRes = MIRBuilder.buildSub(Ty, LHS, RHS); 2859 auto ZExtBorrowIn = MIRBuilder.buildZExt(Ty, BorrowIn); 2860 MIRBuilder.buildSub(Res, TmpRes, ZExtBorrowIn); 2861 2862 auto LHS_EQ_RHS = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, CondTy, LHS, RHS); 2863 auto LHS_ULT_RHS = MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CondTy, LHS, RHS); 2864 MIRBuilder.buildSelect(BorrowOut, LHS_EQ_RHS, BorrowIn, LHS_ULT_RHS); 2865 2866 MI.eraseFromParent(); 2867 return Legalized; 2868 } 2869 case G_UITOFP: 2870 return lowerUITOFP(MI, TypeIdx, Ty); 2871 case G_SITOFP: 2872 return lowerSITOFP(MI, TypeIdx, Ty); 2873 case G_FPTOUI: 2874 return lowerFPTOUI(MI, TypeIdx, Ty); 2875 case G_FPTOSI: 2876 return lowerFPTOSI(MI); 2877 case G_FPTRUNC: 2878 return lowerFPTRUNC(MI, TypeIdx, Ty); 2879 case G_FPOWI: 2880 return lowerFPOWI(MI); 2881 case G_SMIN: 2882 case G_SMAX: 2883 case G_UMIN: 2884 case G_UMAX: 2885 return lowerMinMax(MI, TypeIdx, Ty); 2886 case G_FCOPYSIGN: 2887 return lowerFCopySign(MI, TypeIdx, Ty); 2888 case G_FMINNUM: 2889 case G_FMAXNUM: 2890 return lowerFMinNumMaxNum(MI); 2891 case G_MERGE_VALUES: 2892 return lowerMergeValues(MI); 2893 case G_UNMERGE_VALUES: 2894 return lowerUnmergeValues(MI); 2895 case TargetOpcode::G_SEXT_INREG: { 2896 assert(MI.getOperand(2).isImm() && "Expected immediate"); 2897 int64_t SizeInBits = MI.getOperand(2).getImm(); 2898 2899 Register DstReg = MI.getOperand(0).getReg(); 2900 Register SrcReg = MI.getOperand(1).getReg(); 2901 LLT DstTy = MRI.getType(DstReg); 2902 Register TmpRes = MRI.createGenericVirtualRegister(DstTy); 2903 2904 auto MIBSz = MIRBuilder.buildConstant(DstTy, DstTy.getScalarSizeInBits() - SizeInBits); 2905 MIRBuilder.buildShl(TmpRes, SrcReg, MIBSz->getOperand(0)); 2906 MIRBuilder.buildAShr(DstReg, TmpRes, MIBSz->getOperand(0)); 2907 MI.eraseFromParent(); 2908 return Legalized; 2909 } 2910 case G_EXTRACT_VECTOR_ELT: 2911 return lowerExtractVectorElt(MI); 2912 case G_SHUFFLE_VECTOR: 2913 return lowerShuffleVector(MI); 2914 case G_DYN_STACKALLOC: 2915 return lowerDynStackAlloc(MI); 2916 case G_EXTRACT: 2917 return lowerExtract(MI); 2918 case G_INSERT: 2919 return lowerInsert(MI); 2920 case G_BSWAP: 2921 return lowerBswap(MI); 2922 case G_BITREVERSE: 2923 return lowerBitreverse(MI); 2924 case G_READ_REGISTER: 2925 case G_WRITE_REGISTER: 2926 return lowerReadWriteRegister(MI); 2927 case G_UADDSAT: 2928 case G_USUBSAT: { 2929 // Try to make a reasonable guess about which lowering strategy to use. The 2930 // target can override this with custom lowering and calling the 2931 // implementation functions. 2932 LLT Ty = MRI.getType(MI.getOperand(0).getReg()); 2933 if (LI.isLegalOrCustom({G_UMIN, Ty})) 2934 return lowerAddSubSatToMinMax(MI); 2935 return lowerAddSubSatToAddoSubo(MI); 2936 } 2937 case G_SADDSAT: 2938 case G_SSUBSAT: { 2939 LLT Ty = MRI.getType(MI.getOperand(0).getReg()); 2940 2941 // FIXME: It would probably make more sense to see if G_SADDO is preferred, 2942 // since it's a shorter expansion. However, we would need to figure out the 2943 // preferred boolean type for the carry out for the query. 2944 if (LI.isLegalOrCustom({G_SMIN, Ty}) && LI.isLegalOrCustom({G_SMAX, Ty})) 2945 return lowerAddSubSatToMinMax(MI); 2946 return lowerAddSubSatToAddoSubo(MI); 2947 } 2948 } 2949 } 2950 2951 Align LegalizerHelper::getStackTemporaryAlignment(LLT Ty, 2952 Align MinAlign) const { 2953 // FIXME: We're missing a way to go back from LLT to llvm::Type to query the 2954 // datalayout for the preferred alignment. Also there should be a target hook 2955 // for this to allow targets to reduce the alignment and ignore the 2956 // datalayout. e.g. AMDGPU should always use a 4-byte alignment, regardless of 2957 // the type. 2958 return std::max(Align(PowerOf2Ceil(Ty.getSizeInBytes())), MinAlign); 2959 } 2960 2961 MachineInstrBuilder 2962 LegalizerHelper::createStackTemporary(TypeSize Bytes, Align Alignment, 2963 MachinePointerInfo &PtrInfo) { 2964 MachineFunction &MF = MIRBuilder.getMF(); 2965 const DataLayout &DL = MIRBuilder.getDataLayout(); 2966 int FrameIdx = MF.getFrameInfo().CreateStackObject(Bytes, Alignment, false); 2967 2968 unsigned AddrSpace = DL.getAllocaAddrSpace(); 2969 LLT FramePtrTy = LLT::pointer(AddrSpace, DL.getPointerSizeInBits(AddrSpace)); 2970 2971 PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIdx); 2972 return MIRBuilder.buildFrameIndex(FramePtrTy, FrameIdx); 2973 } 2974 2975 static Register clampDynamicVectorIndex(MachineIRBuilder &B, Register IdxReg, 2976 LLT VecTy) { 2977 int64_t IdxVal; 2978 if (mi_match(IdxReg, *B.getMRI(), m_ICst(IdxVal))) 2979 return IdxReg; 2980 2981 LLT IdxTy = B.getMRI()->getType(IdxReg); 2982 unsigned NElts = VecTy.getNumElements(); 2983 if (isPowerOf2_32(NElts)) { 2984 APInt Imm = APInt::getLowBitsSet(IdxTy.getSizeInBits(), Log2_32(NElts)); 2985 return B.buildAnd(IdxTy, IdxReg, B.buildConstant(IdxTy, Imm)).getReg(0); 2986 } 2987 2988 return B.buildUMin(IdxTy, IdxReg, B.buildConstant(IdxTy, NElts - 1)) 2989 .getReg(0); 2990 } 2991 2992 Register LegalizerHelper::getVectorElementPointer(Register VecPtr, LLT VecTy, 2993 Register Index) { 2994 LLT EltTy = VecTy.getElementType(); 2995 2996 // Calculate the element offset and add it to the pointer. 2997 unsigned EltSize = EltTy.getSizeInBits() / 8; // FIXME: should be ABI size. 2998 assert(EltSize * 8 == EltTy.getSizeInBits() && 2999 "Converting bits to bytes lost precision"); 3000 3001 Index = clampDynamicVectorIndex(MIRBuilder, Index, VecTy); 3002 3003 LLT IdxTy = MRI.getType(Index); 3004 auto Mul = MIRBuilder.buildMul(IdxTy, Index, 3005 MIRBuilder.buildConstant(IdxTy, EltSize)); 3006 3007 LLT PtrTy = MRI.getType(VecPtr); 3008 return MIRBuilder.buildPtrAdd(PtrTy, VecPtr, Mul).getReg(0); 3009 } 3010 3011 LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorImplicitDef( 3012 MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) { 3013 Register DstReg = MI.getOperand(0).getReg(); 3014 LLT DstTy = MRI.getType(DstReg); 3015 LLT LCMTy = getLCMType(DstTy, NarrowTy); 3016 3017 unsigned NumParts = LCMTy.getSizeInBits() / NarrowTy.getSizeInBits(); 3018 3019 auto NewUndef = MIRBuilder.buildUndef(NarrowTy); 3020 SmallVector<Register, 8> Parts(NumParts, NewUndef.getReg(0)); 3021 3022 buildWidenedRemergeToDst(DstReg, LCMTy, Parts); 3023 MI.eraseFromParent(); 3024 return Legalized; 3025 } 3026 3027 // Handle splitting vector operations which need to have the same number of 3028 // elements in each type index, but each type index may have a different element 3029 // type. 3030 // 3031 // e.g. <4 x s64> = G_SHL <4 x s64>, <4 x s32> -> 3032 // <2 x s64> = G_SHL <2 x s64>, <2 x s32> 3033 // <2 x s64> = G_SHL <2 x s64>, <2 x s32> 3034 // 3035 // Also handles some irregular breakdown cases, e.g. 3036 // e.g. <3 x s64> = G_SHL <3 x s64>, <3 x s32> -> 3037 // <2 x s64> = G_SHL <2 x s64>, <2 x s32> 3038 // s64 = G_SHL s64, s32 3039 LegalizerHelper::LegalizeResult 3040 LegalizerHelper::fewerElementsVectorMultiEltType( 3041 MachineInstr &MI, unsigned TypeIdx, LLT NarrowTyArg) { 3042 if (TypeIdx != 0) 3043 return UnableToLegalize; 3044 3045 const LLT NarrowTy0 = NarrowTyArg; 3046 const unsigned NewNumElts = 3047 NarrowTy0.isVector() ? NarrowTy0.getNumElements() : 1; 3048 3049 const Register DstReg = MI.getOperand(0).getReg(); 3050 LLT DstTy = MRI.getType(DstReg); 3051 LLT LeftoverTy0; 3052 3053 // All of the operands need to have the same number of elements, so if we can 3054 // determine a type breakdown for the result type, we can for all of the 3055 // source types. 3056 int NumParts = getNarrowTypeBreakDown(DstTy, NarrowTy0, LeftoverTy0).first; 3057 if (NumParts < 0) 3058 return UnableToLegalize; 3059 3060 SmallVector<MachineInstrBuilder, 4> NewInsts; 3061 3062 SmallVector<Register, 4> DstRegs, LeftoverDstRegs; 3063 SmallVector<Register, 4> PartRegs, LeftoverRegs; 3064 3065 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) { 3066 Register SrcReg = MI.getOperand(I).getReg(); 3067 LLT SrcTyI = MRI.getType(SrcReg); 3068 LLT NarrowTyI = LLT::scalarOrVector(NewNumElts, SrcTyI.getScalarType()); 3069 LLT LeftoverTyI; 3070 3071 // Split this operand into the requested typed registers, and any leftover 3072 // required to reproduce the original type. 3073 if (!extractParts(SrcReg, SrcTyI, NarrowTyI, LeftoverTyI, PartRegs, 3074 LeftoverRegs)) 3075 return UnableToLegalize; 3076 3077 if (I == 1) { 3078 // For the first operand, create an instruction for each part and setup 3079 // the result. 3080 for (Register PartReg : PartRegs) { 3081 Register PartDstReg = MRI.createGenericVirtualRegister(NarrowTy0); 3082 NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode()) 3083 .addDef(PartDstReg) 3084 .addUse(PartReg)); 3085 DstRegs.push_back(PartDstReg); 3086 } 3087 3088 for (Register LeftoverReg : LeftoverRegs) { 3089 Register PartDstReg = MRI.createGenericVirtualRegister(LeftoverTy0); 3090 NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode()) 3091 .addDef(PartDstReg) 3092 .addUse(LeftoverReg)); 3093 LeftoverDstRegs.push_back(PartDstReg); 3094 } 3095 } else { 3096 assert(NewInsts.size() == PartRegs.size() + LeftoverRegs.size()); 3097 3098 // Add the newly created operand splits to the existing instructions. The 3099 // odd-sized pieces are ordered after the requested NarrowTyArg sized 3100 // pieces. 3101 unsigned InstCount = 0; 3102 for (unsigned J = 0, JE = PartRegs.size(); J != JE; ++J) 3103 NewInsts[InstCount++].addUse(PartRegs[J]); 3104 for (unsigned J = 0, JE = LeftoverRegs.size(); J != JE; ++J) 3105 NewInsts[InstCount++].addUse(LeftoverRegs[J]); 3106 } 3107 3108 PartRegs.clear(); 3109 LeftoverRegs.clear(); 3110 } 3111 3112 // Insert the newly built operations and rebuild the result register. 3113 for (auto &MIB : NewInsts) 3114 MIRBuilder.insertInstr(MIB); 3115 3116 insertParts(DstReg, DstTy, NarrowTy0, DstRegs, LeftoverTy0, LeftoverDstRegs); 3117 3118 MI.eraseFromParent(); 3119 return Legalized; 3120 } 3121 3122 LegalizerHelper::LegalizeResult 3123 LegalizerHelper::fewerElementsVectorCasts(MachineInstr &MI, unsigned TypeIdx, 3124 LLT NarrowTy) { 3125 if (TypeIdx != 0) 3126 return UnableToLegalize; 3127 3128 Register DstReg = MI.getOperand(0).getReg(); 3129 Register SrcReg = MI.getOperand(1).getReg(); 3130 LLT DstTy = MRI.getType(DstReg); 3131 LLT SrcTy = MRI.getType(SrcReg); 3132 3133 LLT NarrowTy0 = NarrowTy; 3134 LLT NarrowTy1; 3135 unsigned NumParts; 3136 3137 if (NarrowTy.isVector()) { 3138 // Uneven breakdown not handled. 3139 NumParts = DstTy.getNumElements() / NarrowTy.getNumElements(); 3140 if (NumParts * NarrowTy.getNumElements() != DstTy.getNumElements()) 3141 return UnableToLegalize; 3142 3143 NarrowTy1 = LLT::vector(NumParts, SrcTy.getElementType().getSizeInBits()); 3144 } else { 3145 NumParts = DstTy.getNumElements(); 3146 NarrowTy1 = SrcTy.getElementType(); 3147 } 3148 3149 SmallVector<Register, 4> SrcRegs, DstRegs; 3150 extractParts(SrcReg, NarrowTy1, NumParts, SrcRegs); 3151 3152 for (unsigned I = 0; I < NumParts; ++I) { 3153 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0); 3154 MachineInstr *NewInst = 3155 MIRBuilder.buildInstr(MI.getOpcode(), {DstReg}, {SrcRegs[I]}); 3156 3157 NewInst->setFlags(MI.getFlags()); 3158 DstRegs.push_back(DstReg); 3159 } 3160 3161 if (NarrowTy.isVector()) 3162 MIRBuilder.buildConcatVectors(DstReg, DstRegs); 3163 else 3164 MIRBuilder.buildBuildVector(DstReg, DstRegs); 3165 3166 MI.eraseFromParent(); 3167 return Legalized; 3168 } 3169 3170 LegalizerHelper::LegalizeResult 3171 LegalizerHelper::fewerElementsVectorCmp(MachineInstr &MI, unsigned TypeIdx, 3172 LLT NarrowTy) { 3173 Register DstReg = MI.getOperand(0).getReg(); 3174 Register Src0Reg = MI.getOperand(2).getReg(); 3175 LLT DstTy = MRI.getType(DstReg); 3176 LLT SrcTy = MRI.getType(Src0Reg); 3177 3178 unsigned NumParts; 3179 LLT NarrowTy0, NarrowTy1; 3180 3181 if (TypeIdx == 0) { 3182 unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1; 3183 unsigned OldElts = DstTy.getNumElements(); 3184 3185 NarrowTy0 = NarrowTy; 3186 NumParts = NarrowTy.isVector() ? (OldElts / NewElts) : DstTy.getNumElements(); 3187 NarrowTy1 = NarrowTy.isVector() ? 3188 LLT::vector(NarrowTy.getNumElements(), SrcTy.getScalarSizeInBits()) : 3189 SrcTy.getElementType(); 3190 3191 } else { 3192 unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1; 3193 unsigned OldElts = SrcTy.getNumElements(); 3194 3195 NumParts = NarrowTy.isVector() ? (OldElts / NewElts) : 3196 NarrowTy.getNumElements(); 3197 NarrowTy0 = LLT::vector(NarrowTy.getNumElements(), 3198 DstTy.getScalarSizeInBits()); 3199 NarrowTy1 = NarrowTy; 3200 } 3201 3202 // FIXME: Don't know how to handle the situation where the small vectors 3203 // aren't all the same size yet. 3204 if (NarrowTy1.isVector() && 3205 NarrowTy1.getNumElements() * NumParts != DstTy.getNumElements()) 3206 return UnableToLegalize; 3207 3208 CmpInst::Predicate Pred 3209 = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate()); 3210 3211 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs; 3212 extractParts(MI.getOperand(2).getReg(), NarrowTy1, NumParts, Src1Regs); 3213 extractParts(MI.getOperand(3).getReg(), NarrowTy1, NumParts, Src2Regs); 3214 3215 for (unsigned I = 0; I < NumParts; ++I) { 3216 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0); 3217 DstRegs.push_back(DstReg); 3218 3219 if (MI.getOpcode() == TargetOpcode::G_ICMP) 3220 MIRBuilder.buildICmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]); 3221 else { 3222 MachineInstr *NewCmp 3223 = MIRBuilder.buildFCmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]); 3224 NewCmp->setFlags(MI.getFlags()); 3225 } 3226 } 3227 3228 if (NarrowTy1.isVector()) 3229 MIRBuilder.buildConcatVectors(DstReg, DstRegs); 3230 else 3231 MIRBuilder.buildBuildVector(DstReg, DstRegs); 3232 3233 MI.eraseFromParent(); 3234 return Legalized; 3235 } 3236 3237 LegalizerHelper::LegalizeResult 3238 LegalizerHelper::fewerElementsVectorSelect(MachineInstr &MI, unsigned TypeIdx, 3239 LLT NarrowTy) { 3240 Register DstReg = MI.getOperand(0).getReg(); 3241 Register CondReg = MI.getOperand(1).getReg(); 3242 3243 unsigned NumParts = 0; 3244 LLT NarrowTy0, NarrowTy1; 3245 3246 LLT DstTy = MRI.getType(DstReg); 3247 LLT CondTy = MRI.getType(CondReg); 3248 unsigned Size = DstTy.getSizeInBits(); 3249 3250 assert(TypeIdx == 0 || CondTy.isVector()); 3251 3252 if (TypeIdx == 0) { 3253 NarrowTy0 = NarrowTy; 3254 NarrowTy1 = CondTy; 3255 3256 unsigned NarrowSize = NarrowTy0.getSizeInBits(); 3257 // FIXME: Don't know how to handle the situation where the small vectors 3258 // aren't all the same size yet. 3259 if (Size % NarrowSize != 0) 3260 return UnableToLegalize; 3261 3262 NumParts = Size / NarrowSize; 3263 3264 // Need to break down the condition type 3265 if (CondTy.isVector()) { 3266 if (CondTy.getNumElements() == NumParts) 3267 NarrowTy1 = CondTy.getElementType(); 3268 else 3269 NarrowTy1 = LLT::vector(CondTy.getNumElements() / NumParts, 3270 CondTy.getScalarSizeInBits()); 3271 } 3272 } else { 3273 NumParts = CondTy.getNumElements(); 3274 if (NarrowTy.isVector()) { 3275 // TODO: Handle uneven breakdown. 3276 if (NumParts * NarrowTy.getNumElements() != CondTy.getNumElements()) 3277 return UnableToLegalize; 3278 3279 return UnableToLegalize; 3280 } else { 3281 NarrowTy0 = DstTy.getElementType(); 3282 NarrowTy1 = NarrowTy; 3283 } 3284 } 3285 3286 SmallVector<Register, 2> DstRegs, Src0Regs, Src1Regs, Src2Regs; 3287 if (CondTy.isVector()) 3288 extractParts(MI.getOperand(1).getReg(), NarrowTy1, NumParts, Src0Regs); 3289 3290 extractParts(MI.getOperand(2).getReg(), NarrowTy0, NumParts, Src1Regs); 3291 extractParts(MI.getOperand(3).getReg(), NarrowTy0, NumParts, Src2Regs); 3292 3293 for (unsigned i = 0; i < NumParts; ++i) { 3294 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0); 3295 MIRBuilder.buildSelect(DstReg, CondTy.isVector() ? Src0Regs[i] : CondReg, 3296 Src1Regs[i], Src2Regs[i]); 3297 DstRegs.push_back(DstReg); 3298 } 3299 3300 if (NarrowTy0.isVector()) 3301 MIRBuilder.buildConcatVectors(DstReg, DstRegs); 3302 else 3303 MIRBuilder.buildBuildVector(DstReg, DstRegs); 3304 3305 MI.eraseFromParent(); 3306 return Legalized; 3307 } 3308 3309 LegalizerHelper::LegalizeResult 3310 LegalizerHelper::fewerElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx, 3311 LLT NarrowTy) { 3312 const Register DstReg = MI.getOperand(0).getReg(); 3313 LLT PhiTy = MRI.getType(DstReg); 3314 LLT LeftoverTy; 3315 3316 // All of the operands need to have the same number of elements, so if we can 3317 // determine a type breakdown for the result type, we can for all of the 3318 // source types. 3319 int NumParts, NumLeftover; 3320 std::tie(NumParts, NumLeftover) 3321 = getNarrowTypeBreakDown(PhiTy, NarrowTy, LeftoverTy); 3322 if (NumParts < 0) 3323 return UnableToLegalize; 3324 3325 SmallVector<Register, 4> DstRegs, LeftoverDstRegs; 3326 SmallVector<MachineInstrBuilder, 4> NewInsts; 3327 3328 const int TotalNumParts = NumParts + NumLeftover; 3329 3330 // Insert the new phis in the result block first. 3331 for (int I = 0; I != TotalNumParts; ++I) { 3332 LLT Ty = I < NumParts ? NarrowTy : LeftoverTy; 3333 Register PartDstReg = MRI.createGenericVirtualRegister(Ty); 3334 NewInsts.push_back(MIRBuilder.buildInstr(TargetOpcode::G_PHI) 3335 .addDef(PartDstReg)); 3336 if (I < NumParts) 3337 DstRegs.push_back(PartDstReg); 3338 else 3339 LeftoverDstRegs.push_back(PartDstReg); 3340 } 3341 3342 MachineBasicBlock *MBB = MI.getParent(); 3343 MIRBuilder.setInsertPt(*MBB, MBB->getFirstNonPHI()); 3344 insertParts(DstReg, PhiTy, NarrowTy, DstRegs, LeftoverTy, LeftoverDstRegs); 3345 3346 SmallVector<Register, 4> PartRegs, LeftoverRegs; 3347 3348 // Insert code to extract the incoming values in each predecessor block. 3349 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) { 3350 PartRegs.clear(); 3351 LeftoverRegs.clear(); 3352 3353 Register SrcReg = MI.getOperand(I).getReg(); 3354 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB(); 3355 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator()); 3356 3357 LLT Unused; 3358 if (!extractParts(SrcReg, PhiTy, NarrowTy, Unused, PartRegs, 3359 LeftoverRegs)) 3360 return UnableToLegalize; 3361 3362 // Add the newly created operand splits to the existing instructions. The 3363 // odd-sized pieces are ordered after the requested NarrowTyArg sized 3364 // pieces. 3365 for (int J = 0; J != TotalNumParts; ++J) { 3366 MachineInstrBuilder MIB = NewInsts[J]; 3367 MIB.addUse(J < NumParts ? PartRegs[J] : LeftoverRegs[J - NumParts]); 3368 MIB.addMBB(&OpMBB); 3369 } 3370 } 3371 3372 MI.eraseFromParent(); 3373 return Legalized; 3374 } 3375 3376 LegalizerHelper::LegalizeResult 3377 LegalizerHelper::fewerElementsVectorUnmergeValues(MachineInstr &MI, 3378 unsigned TypeIdx, 3379 LLT NarrowTy) { 3380 if (TypeIdx != 1) 3381 return UnableToLegalize; 3382 3383 const int NumDst = MI.getNumOperands() - 1; 3384 const Register SrcReg = MI.getOperand(NumDst).getReg(); 3385 LLT SrcTy = MRI.getType(SrcReg); 3386 3387 LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); 3388 3389 // TODO: Create sequence of extracts. 3390 if (DstTy == NarrowTy) 3391 return UnableToLegalize; 3392 3393 LLT GCDTy = getGCDType(SrcTy, NarrowTy); 3394 if (DstTy == GCDTy) { 3395 // This would just be a copy of the same unmerge. 3396 // TODO: Create extracts, pad with undef and create intermediate merges. 3397 return UnableToLegalize; 3398 } 3399 3400 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg); 3401 const int NumUnmerge = Unmerge->getNumOperands() - 1; 3402 const int PartsPerUnmerge = NumDst / NumUnmerge; 3403 3404 for (int I = 0; I != NumUnmerge; ++I) { 3405 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES); 3406 3407 for (int J = 0; J != PartsPerUnmerge; ++J) 3408 MIB.addDef(MI.getOperand(I * PartsPerUnmerge + J).getReg()); 3409 MIB.addUse(Unmerge.getReg(I)); 3410 } 3411 3412 MI.eraseFromParent(); 3413 return Legalized; 3414 } 3415 3416 LegalizerHelper::LegalizeResult 3417 LegalizerHelper::fewerElementsVectorBuildVector(MachineInstr &MI, 3418 unsigned TypeIdx, 3419 LLT NarrowTy) { 3420 assert(TypeIdx == 0 && "not a vector type index"); 3421 Register DstReg = MI.getOperand(0).getReg(); 3422 LLT DstTy = MRI.getType(DstReg); 3423 LLT SrcTy = DstTy.getElementType(); 3424 3425 int DstNumElts = DstTy.getNumElements(); 3426 int NarrowNumElts = NarrowTy.getNumElements(); 3427 int NumConcat = (DstNumElts + NarrowNumElts - 1) / NarrowNumElts; 3428 LLT WidenedDstTy = LLT::vector(NarrowNumElts * NumConcat, SrcTy); 3429 3430 SmallVector<Register, 8> ConcatOps; 3431 SmallVector<Register, 8> SubBuildVector; 3432 3433 Register UndefReg; 3434 if (WidenedDstTy != DstTy) 3435 UndefReg = MIRBuilder.buildUndef(SrcTy).getReg(0); 3436 3437 // Create a G_CONCAT_VECTORS of NarrowTy pieces, padding with undef as 3438 // necessary. 3439 // 3440 // %3:_(<3 x s16>) = G_BUILD_VECTOR %0, %1, %2 3441 // -> <2 x s16> 3442 // 3443 // %4:_(s16) = G_IMPLICIT_DEF 3444 // %5:_(<2 x s16>) = G_BUILD_VECTOR %0, %1 3445 // %6:_(<2 x s16>) = G_BUILD_VECTOR %2, %4 3446 // %7:_(<4 x s16>) = G_CONCAT_VECTORS %5, %6 3447 // %3:_(<3 x s16>) = G_EXTRACT %7, 0 3448 for (int I = 0; I != NumConcat; ++I) { 3449 for (int J = 0; J != NarrowNumElts; ++J) { 3450 int SrcIdx = NarrowNumElts * I + J; 3451 3452 if (SrcIdx < DstNumElts) { 3453 Register SrcReg = MI.getOperand(SrcIdx + 1).getReg(); 3454 SubBuildVector.push_back(SrcReg); 3455 } else 3456 SubBuildVector.push_back(UndefReg); 3457 } 3458 3459 auto BuildVec = MIRBuilder.buildBuildVector(NarrowTy, SubBuildVector); 3460 ConcatOps.push_back(BuildVec.getReg(0)); 3461 SubBuildVector.clear(); 3462 } 3463 3464 if (DstTy == WidenedDstTy) 3465 MIRBuilder.buildConcatVectors(DstReg, ConcatOps); 3466 else { 3467 auto Concat = MIRBuilder.buildConcatVectors(WidenedDstTy, ConcatOps); 3468 MIRBuilder.buildExtract(DstReg, Concat, 0); 3469 } 3470 3471 MI.eraseFromParent(); 3472 return Legalized; 3473 } 3474 3475 LegalizerHelper::LegalizeResult 3476 LegalizerHelper::fewerElementsVectorExtractVectorElt(MachineInstr &MI, 3477 unsigned TypeIdx, 3478 LLT NarrowVecTy) { 3479 assert(TypeIdx == 1 && "not a vector type index"); 3480 3481 // TODO: Handle total scalarization case. 3482 if (!NarrowVecTy.isVector()) 3483 return UnableToLegalize; 3484 3485 Register DstReg = MI.getOperand(0).getReg(); 3486 Register SrcVec = MI.getOperand(1).getReg(); 3487 Register Idx = MI.getOperand(2).getReg(); 3488 LLT VecTy = MRI.getType(SrcVec); 3489 3490 // If the index is a constant, we can really break this down as you would 3491 // expect, and index into the target size pieces. 3492 int64_t IdxVal; 3493 if (mi_match(Idx, MRI, m_ICst(IdxVal))) { 3494 // Avoid out of bounds indexing the pieces. 3495 if (IdxVal >= VecTy.getNumElements()) { 3496 MIRBuilder.buildUndef(DstReg); 3497 MI.eraseFromParent(); 3498 return Legalized; 3499 } 3500 3501 SmallVector<Register, 8> VecParts; 3502 LLT GCDTy = extractGCDType(VecParts, VecTy, NarrowVecTy, SrcVec); 3503 3504 // Build a sequence of NarrowTy pieces in VecParts for this operand. 3505 buildLCMMergePieces(VecTy, NarrowVecTy, GCDTy, VecParts, 3506 TargetOpcode::G_ANYEXT); 3507 3508 unsigned NewNumElts = NarrowVecTy.getNumElements(); 3509 3510 LLT IdxTy = MRI.getType(Idx); 3511 int64_t PartIdx = IdxVal / NewNumElts; 3512 auto NewIdx = 3513 MIRBuilder.buildConstant(IdxTy, IdxVal - NewNumElts * PartIdx); 3514 3515 MIRBuilder.buildExtractVectorElement(DstReg, VecParts[PartIdx], NewIdx); 3516 MI.eraseFromParent(); 3517 return Legalized; 3518 } 3519 3520 // With a variable index, we can't perform the extract in a smaller type, so 3521 // we're forced to expand this. 3522 // 3523 // TODO: We could emit a chain of compare/select to figure out which piece to 3524 // index. 3525 return lowerExtractVectorElt(MI); 3526 } 3527 3528 LegalizerHelper::LegalizeResult 3529 LegalizerHelper::reduceLoadStoreWidth(MachineInstr &MI, unsigned TypeIdx, 3530 LLT NarrowTy) { 3531 // FIXME: Don't know how to handle secondary types yet. 3532 if (TypeIdx != 0) 3533 return UnableToLegalize; 3534 3535 MachineMemOperand *MMO = *MI.memoperands_begin(); 3536 3537 // This implementation doesn't work for atomics. Give up instead of doing 3538 // something invalid. 3539 if (MMO->getOrdering() != AtomicOrdering::NotAtomic || 3540 MMO->getFailureOrdering() != AtomicOrdering::NotAtomic) 3541 return UnableToLegalize; 3542 3543 bool IsLoad = MI.getOpcode() == TargetOpcode::G_LOAD; 3544 Register ValReg = MI.getOperand(0).getReg(); 3545 Register AddrReg = MI.getOperand(1).getReg(); 3546 LLT ValTy = MRI.getType(ValReg); 3547 3548 // FIXME: Do we need a distinct NarrowMemory legalize action? 3549 if (ValTy.getSizeInBits() != 8 * MMO->getSize()) { 3550 LLVM_DEBUG(dbgs() << "Can't narrow extload/truncstore\n"); 3551 return UnableToLegalize; 3552 } 3553 3554 int NumParts = -1; 3555 int NumLeftover = -1; 3556 LLT LeftoverTy; 3557 SmallVector<Register, 8> NarrowRegs, NarrowLeftoverRegs; 3558 if (IsLoad) { 3559 std::tie(NumParts, NumLeftover) = getNarrowTypeBreakDown(ValTy, NarrowTy, LeftoverTy); 3560 } else { 3561 if (extractParts(ValReg, ValTy, NarrowTy, LeftoverTy, NarrowRegs, 3562 NarrowLeftoverRegs)) { 3563 NumParts = NarrowRegs.size(); 3564 NumLeftover = NarrowLeftoverRegs.size(); 3565 } 3566 } 3567 3568 if (NumParts == -1) 3569 return UnableToLegalize; 3570 3571 LLT PtrTy = MRI.getType(AddrReg); 3572 const LLT OffsetTy = LLT::scalar(PtrTy.getSizeInBits()); 3573 3574 unsigned TotalSize = ValTy.getSizeInBits(); 3575 3576 // Split the load/store into PartTy sized pieces starting at Offset. If this 3577 // is a load, return the new registers in ValRegs. For a store, each elements 3578 // of ValRegs should be PartTy. Returns the next offset that needs to be 3579 // handled. 3580 auto splitTypePieces = [=](LLT PartTy, SmallVectorImpl<Register> &ValRegs, 3581 unsigned Offset) -> unsigned { 3582 MachineFunction &MF = MIRBuilder.getMF(); 3583 unsigned PartSize = PartTy.getSizeInBits(); 3584 for (unsigned Idx = 0, E = NumParts; Idx != E && Offset < TotalSize; 3585 Offset += PartSize, ++Idx) { 3586 unsigned ByteSize = PartSize / 8; 3587 unsigned ByteOffset = Offset / 8; 3588 Register NewAddrReg; 3589 3590 MIRBuilder.materializePtrAdd(NewAddrReg, AddrReg, OffsetTy, ByteOffset); 3591 3592 MachineMemOperand *NewMMO = 3593 MF.getMachineMemOperand(MMO, ByteOffset, ByteSize); 3594 3595 if (IsLoad) { 3596 Register Dst = MRI.createGenericVirtualRegister(PartTy); 3597 ValRegs.push_back(Dst); 3598 MIRBuilder.buildLoad(Dst, NewAddrReg, *NewMMO); 3599 } else { 3600 MIRBuilder.buildStore(ValRegs[Idx], NewAddrReg, *NewMMO); 3601 } 3602 } 3603 3604 return Offset; 3605 }; 3606 3607 unsigned HandledOffset = splitTypePieces(NarrowTy, NarrowRegs, 0); 3608 3609 // Handle the rest of the register if this isn't an even type breakdown. 3610 if (LeftoverTy.isValid()) 3611 splitTypePieces(LeftoverTy, NarrowLeftoverRegs, HandledOffset); 3612 3613 if (IsLoad) { 3614 insertParts(ValReg, ValTy, NarrowTy, NarrowRegs, 3615 LeftoverTy, NarrowLeftoverRegs); 3616 } 3617 3618 MI.eraseFromParent(); 3619 return Legalized; 3620 } 3621 3622 LegalizerHelper::LegalizeResult 3623 LegalizerHelper::reduceOperationWidth(MachineInstr &MI, unsigned int TypeIdx, 3624 LLT NarrowTy) { 3625 assert(TypeIdx == 0 && "only one type index expected"); 3626 3627 const unsigned Opc = MI.getOpcode(); 3628 const int NumOps = MI.getNumOperands() - 1; 3629 const Register DstReg = MI.getOperand(0).getReg(); 3630 const unsigned Flags = MI.getFlags(); 3631 const unsigned NarrowSize = NarrowTy.getSizeInBits(); 3632 const LLT NarrowScalarTy = LLT::scalar(NarrowSize); 3633 3634 assert(NumOps <= 3 && "expected instruction with 1 result and 1-3 sources"); 3635 3636 // First of all check whether we are narrowing (changing the element type) 3637 // or reducing the vector elements 3638 const LLT DstTy = MRI.getType(DstReg); 3639 const bool IsNarrow = NarrowTy.getScalarType() != DstTy.getScalarType(); 3640 3641 SmallVector<Register, 8> ExtractedRegs[3]; 3642 SmallVector<Register, 8> Parts; 3643 3644 unsigned NarrowElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1; 3645 3646 // Break down all the sources into NarrowTy pieces we can operate on. This may 3647 // involve creating merges to a wider type, padded with undef. 3648 for (int I = 0; I != NumOps; ++I) { 3649 Register SrcReg = MI.getOperand(I + 1).getReg(); 3650 LLT SrcTy = MRI.getType(SrcReg); 3651 3652 // The type to narrow SrcReg to. For narrowing, this is a smaller scalar. 3653 // For fewerElements, this is a smaller vector with the same element type. 3654 LLT OpNarrowTy; 3655 if (IsNarrow) { 3656 OpNarrowTy = NarrowScalarTy; 3657 3658 // In case of narrowing, we need to cast vectors to scalars for this to 3659 // work properly 3660 // FIXME: Can we do without the bitcast here if we're narrowing? 3661 if (SrcTy.isVector()) { 3662 SrcTy = LLT::scalar(SrcTy.getSizeInBits()); 3663 SrcReg = MIRBuilder.buildBitcast(SrcTy, SrcReg).getReg(0); 3664 } 3665 } else { 3666 OpNarrowTy = LLT::scalarOrVector(NarrowElts, SrcTy.getScalarType()); 3667 } 3668 3669 LLT GCDTy = extractGCDType(ExtractedRegs[I], SrcTy, OpNarrowTy, SrcReg); 3670 3671 // Build a sequence of NarrowTy pieces in ExtractedRegs for this operand. 3672 buildLCMMergePieces(SrcTy, OpNarrowTy, GCDTy, ExtractedRegs[I], 3673 TargetOpcode::G_ANYEXT); 3674 } 3675 3676 SmallVector<Register, 8> ResultRegs; 3677 3678 // Input operands for each sub-instruction. 3679 SmallVector<SrcOp, 4> InputRegs(NumOps, Register()); 3680 3681 int NumParts = ExtractedRegs[0].size(); 3682 const unsigned DstSize = DstTy.getSizeInBits(); 3683 const LLT DstScalarTy = LLT::scalar(DstSize); 3684 3685 // Narrowing needs to use scalar types 3686 LLT DstLCMTy, NarrowDstTy; 3687 if (IsNarrow) { 3688 DstLCMTy = getLCMType(DstScalarTy, NarrowScalarTy); 3689 NarrowDstTy = NarrowScalarTy; 3690 } else { 3691 DstLCMTy = getLCMType(DstTy, NarrowTy); 3692 NarrowDstTy = NarrowTy; 3693 } 3694 3695 // We widened the source registers to satisfy merge/unmerge size 3696 // constraints. We'll have some extra fully undef parts. 3697 const int NumRealParts = (DstSize + NarrowSize - 1) / NarrowSize; 3698 3699 for (int I = 0; I != NumRealParts; ++I) { 3700 // Emit this instruction on each of the split pieces. 3701 for (int J = 0; J != NumOps; ++J) 3702 InputRegs[J] = ExtractedRegs[J][I]; 3703 3704 auto Inst = MIRBuilder.buildInstr(Opc, {NarrowDstTy}, InputRegs, Flags); 3705 ResultRegs.push_back(Inst.getReg(0)); 3706 } 3707 3708 // Fill out the widened result with undef instead of creating instructions 3709 // with undef inputs. 3710 int NumUndefParts = NumParts - NumRealParts; 3711 if (NumUndefParts != 0) 3712 ResultRegs.append(NumUndefParts, 3713 MIRBuilder.buildUndef(NarrowDstTy).getReg(0)); 3714 3715 // Extract the possibly padded result. Use a scratch register if we need to do 3716 // a final bitcast, otherwise use the original result register. 3717 Register MergeDstReg; 3718 if (IsNarrow && DstTy.isVector()) 3719 MergeDstReg = MRI.createGenericVirtualRegister(DstScalarTy); 3720 else 3721 MergeDstReg = DstReg; 3722 3723 buildWidenedRemergeToDst(MergeDstReg, DstLCMTy, ResultRegs); 3724 3725 // Recast to vector if we narrowed a vector 3726 if (IsNarrow && DstTy.isVector()) 3727 MIRBuilder.buildBitcast(DstReg, MergeDstReg); 3728 3729 MI.eraseFromParent(); 3730 return Legalized; 3731 } 3732 3733 LegalizerHelper::LegalizeResult 3734 LegalizerHelper::fewerElementsVectorSextInReg(MachineInstr &MI, unsigned TypeIdx, 3735 LLT NarrowTy) { 3736 Register DstReg = MI.getOperand(0).getReg(); 3737 Register SrcReg = MI.getOperand(1).getReg(); 3738 int64_t Imm = MI.getOperand(2).getImm(); 3739 3740 LLT DstTy = MRI.getType(DstReg); 3741 3742 SmallVector<Register, 8> Parts; 3743 LLT GCDTy = extractGCDType(Parts, DstTy, NarrowTy, SrcReg); 3744 LLT LCMTy = buildLCMMergePieces(DstTy, NarrowTy, GCDTy, Parts); 3745 3746 for (Register &R : Parts) 3747 R = MIRBuilder.buildSExtInReg(NarrowTy, R, Imm).getReg(0); 3748 3749 buildWidenedRemergeToDst(DstReg, LCMTy, Parts); 3750 3751 MI.eraseFromParent(); 3752 return Legalized; 3753 } 3754 3755 LegalizerHelper::LegalizeResult 3756 LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx, 3757 LLT NarrowTy) { 3758 using namespace TargetOpcode; 3759 3760 switch (MI.getOpcode()) { 3761 case G_IMPLICIT_DEF: 3762 return fewerElementsVectorImplicitDef(MI, TypeIdx, NarrowTy); 3763 case G_TRUNC: 3764 case G_AND: 3765 case G_OR: 3766 case G_XOR: 3767 case G_ADD: 3768 case G_SUB: 3769 case G_MUL: 3770 case G_PTR_ADD: 3771 case G_SMULH: 3772 case G_UMULH: 3773 case G_FADD: 3774 case G_FMUL: 3775 case G_FSUB: 3776 case G_FNEG: 3777 case G_FABS: 3778 case G_FCANONICALIZE: 3779 case G_FDIV: 3780 case G_FREM: 3781 case G_FMA: 3782 case G_FMAD: 3783 case G_FPOW: 3784 case G_FEXP: 3785 case G_FEXP2: 3786 case G_FLOG: 3787 case G_FLOG2: 3788 case G_FLOG10: 3789 case G_FNEARBYINT: 3790 case G_FCEIL: 3791 case G_FFLOOR: 3792 case G_FRINT: 3793 case G_INTRINSIC_ROUND: 3794 case G_INTRINSIC_ROUNDEVEN: 3795 case G_INTRINSIC_TRUNC: 3796 case G_FCOS: 3797 case G_FSIN: 3798 case G_FSQRT: 3799 case G_BSWAP: 3800 case G_BITREVERSE: 3801 case G_SDIV: 3802 case G_UDIV: 3803 case G_SREM: 3804 case G_UREM: 3805 case G_SMIN: 3806 case G_SMAX: 3807 case G_UMIN: 3808 case G_UMAX: 3809 case G_FMINNUM: 3810 case G_FMAXNUM: 3811 case G_FMINNUM_IEEE: 3812 case G_FMAXNUM_IEEE: 3813 case G_FMINIMUM: 3814 case G_FMAXIMUM: 3815 case G_FSHL: 3816 case G_FSHR: 3817 case G_FREEZE: 3818 case G_SADDSAT: 3819 case G_SSUBSAT: 3820 case G_UADDSAT: 3821 case G_USUBSAT: 3822 return reduceOperationWidth(MI, TypeIdx, NarrowTy); 3823 case G_SHL: 3824 case G_LSHR: 3825 case G_ASHR: 3826 case G_CTLZ: 3827 case G_CTLZ_ZERO_UNDEF: 3828 case G_CTTZ: 3829 case G_CTTZ_ZERO_UNDEF: 3830 case G_CTPOP: 3831 case G_FCOPYSIGN: 3832 return fewerElementsVectorMultiEltType(MI, TypeIdx, NarrowTy); 3833 case G_ZEXT: 3834 case G_SEXT: 3835 case G_ANYEXT: 3836 case G_FPEXT: 3837 case G_FPTRUNC: 3838 case G_SITOFP: 3839 case G_UITOFP: 3840 case G_FPTOSI: 3841 case G_FPTOUI: 3842 case G_INTTOPTR: 3843 case G_PTRTOINT: 3844 case G_ADDRSPACE_CAST: 3845 return fewerElementsVectorCasts(MI, TypeIdx, NarrowTy); 3846 case G_ICMP: 3847 case G_FCMP: 3848 return fewerElementsVectorCmp(MI, TypeIdx, NarrowTy); 3849 case G_SELECT: 3850 return fewerElementsVectorSelect(MI, TypeIdx, NarrowTy); 3851 case G_PHI: 3852 return fewerElementsVectorPhi(MI, TypeIdx, NarrowTy); 3853 case G_UNMERGE_VALUES: 3854 return fewerElementsVectorUnmergeValues(MI, TypeIdx, NarrowTy); 3855 case G_BUILD_VECTOR: 3856 return fewerElementsVectorBuildVector(MI, TypeIdx, NarrowTy); 3857 case G_EXTRACT_VECTOR_ELT: 3858 return fewerElementsVectorExtractVectorElt(MI, TypeIdx, NarrowTy); 3859 case G_LOAD: 3860 case G_STORE: 3861 return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy); 3862 case G_SEXT_INREG: 3863 return fewerElementsVectorSextInReg(MI, TypeIdx, NarrowTy); 3864 default: 3865 return UnableToLegalize; 3866 } 3867 } 3868 3869 LegalizerHelper::LegalizeResult 3870 LegalizerHelper::narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt, 3871 const LLT HalfTy, const LLT AmtTy) { 3872 3873 Register InL = MRI.createGenericVirtualRegister(HalfTy); 3874 Register InH = MRI.createGenericVirtualRegister(HalfTy); 3875 MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1)); 3876 3877 if (Amt.isNullValue()) { 3878 MIRBuilder.buildMerge(MI.getOperand(0), {InL, InH}); 3879 MI.eraseFromParent(); 3880 return Legalized; 3881 } 3882 3883 LLT NVT = HalfTy; 3884 unsigned NVTBits = HalfTy.getSizeInBits(); 3885 unsigned VTBits = 2 * NVTBits; 3886 3887 SrcOp Lo(Register(0)), Hi(Register(0)); 3888 if (MI.getOpcode() == TargetOpcode::G_SHL) { 3889 if (Amt.ugt(VTBits)) { 3890 Lo = Hi = MIRBuilder.buildConstant(NVT, 0); 3891 } else if (Amt.ugt(NVTBits)) { 3892 Lo = MIRBuilder.buildConstant(NVT, 0); 3893 Hi = MIRBuilder.buildShl(NVT, InL, 3894 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits)); 3895 } else if (Amt == NVTBits) { 3896 Lo = MIRBuilder.buildConstant(NVT, 0); 3897 Hi = InL; 3898 } else { 3899 Lo = MIRBuilder.buildShl(NVT, InL, MIRBuilder.buildConstant(AmtTy, Amt)); 3900 auto OrLHS = 3901 MIRBuilder.buildShl(NVT, InH, MIRBuilder.buildConstant(AmtTy, Amt)); 3902 auto OrRHS = MIRBuilder.buildLShr( 3903 NVT, InL, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits)); 3904 Hi = MIRBuilder.buildOr(NVT, OrLHS, OrRHS); 3905 } 3906 } else if (MI.getOpcode() == TargetOpcode::G_LSHR) { 3907 if (Amt.ugt(VTBits)) { 3908 Lo = Hi = MIRBuilder.buildConstant(NVT, 0); 3909 } else if (Amt.ugt(NVTBits)) { 3910 Lo = MIRBuilder.buildLShr(NVT, InH, 3911 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits)); 3912 Hi = MIRBuilder.buildConstant(NVT, 0); 3913 } else if (Amt == NVTBits) { 3914 Lo = InH; 3915 Hi = MIRBuilder.buildConstant(NVT, 0); 3916 } else { 3917 auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt); 3918 3919 auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst); 3920 auto OrRHS = MIRBuilder.buildShl( 3921 NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits)); 3922 3923 Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS); 3924 Hi = MIRBuilder.buildLShr(NVT, InH, ShiftAmtConst); 3925 } 3926 } else { 3927 if (Amt.ugt(VTBits)) { 3928 Hi = Lo = MIRBuilder.buildAShr( 3929 NVT, InH, MIRBuilder.buildConstant(AmtTy, NVTBits - 1)); 3930 } else if (Amt.ugt(NVTBits)) { 3931 Lo = MIRBuilder.buildAShr(NVT, InH, 3932 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits)); 3933 Hi = MIRBuilder.buildAShr(NVT, InH, 3934 MIRBuilder.buildConstant(AmtTy, NVTBits - 1)); 3935 } else if (Amt == NVTBits) { 3936 Lo = InH; 3937 Hi = MIRBuilder.buildAShr(NVT, InH, 3938 MIRBuilder.buildConstant(AmtTy, NVTBits - 1)); 3939 } else { 3940 auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt); 3941 3942 auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst); 3943 auto OrRHS = MIRBuilder.buildShl( 3944 NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits)); 3945 3946 Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS); 3947 Hi = MIRBuilder.buildAShr(NVT, InH, ShiftAmtConst); 3948 } 3949 } 3950 3951 MIRBuilder.buildMerge(MI.getOperand(0), {Lo, Hi}); 3952 MI.eraseFromParent(); 3953 3954 return Legalized; 3955 } 3956 3957 // TODO: Optimize if constant shift amount. 3958 LegalizerHelper::LegalizeResult 3959 LegalizerHelper::narrowScalarShift(MachineInstr &MI, unsigned TypeIdx, 3960 LLT RequestedTy) { 3961 if (TypeIdx == 1) { 3962 Observer.changingInstr(MI); 3963 narrowScalarSrc(MI, RequestedTy, 2); 3964 Observer.changedInstr(MI); 3965 return Legalized; 3966 } 3967 3968 Register DstReg = MI.getOperand(0).getReg(); 3969 LLT DstTy = MRI.getType(DstReg); 3970 if (DstTy.isVector()) 3971 return UnableToLegalize; 3972 3973 Register Amt = MI.getOperand(2).getReg(); 3974 LLT ShiftAmtTy = MRI.getType(Amt); 3975 const unsigned DstEltSize = DstTy.getScalarSizeInBits(); 3976 if (DstEltSize % 2 != 0) 3977 return UnableToLegalize; 3978 3979 // Ignore the input type. We can only go to exactly half the size of the 3980 // input. If that isn't small enough, the resulting pieces will be further 3981 // legalized. 3982 const unsigned NewBitSize = DstEltSize / 2; 3983 const LLT HalfTy = LLT::scalar(NewBitSize); 3984 const LLT CondTy = LLT::scalar(1); 3985 3986 if (const MachineInstr *KShiftAmt = 3987 getOpcodeDef(TargetOpcode::G_CONSTANT, Amt, MRI)) { 3988 return narrowScalarShiftByConstant( 3989 MI, KShiftAmt->getOperand(1).getCImm()->getValue(), HalfTy, ShiftAmtTy); 3990 } 3991 3992 // TODO: Expand with known bits. 3993 3994 // Handle the fully general expansion by an unknown amount. 3995 auto NewBits = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize); 3996 3997 Register InL = MRI.createGenericVirtualRegister(HalfTy); 3998 Register InH = MRI.createGenericVirtualRegister(HalfTy); 3999 MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1)); 4000 4001 auto AmtExcess = MIRBuilder.buildSub(ShiftAmtTy, Amt, NewBits); 4002 auto AmtLack = MIRBuilder.buildSub(ShiftAmtTy, NewBits, Amt); 4003 4004 auto Zero = MIRBuilder.buildConstant(ShiftAmtTy, 0); 4005 auto IsShort = MIRBuilder.buildICmp(ICmpInst::ICMP_ULT, CondTy, Amt, NewBits); 4006 auto IsZero = MIRBuilder.buildICmp(ICmpInst::ICMP_EQ, CondTy, Amt, Zero); 4007 4008 Register ResultRegs[2]; 4009 switch (MI.getOpcode()) { 4010 case TargetOpcode::G_SHL: { 4011 // Short: ShAmt < NewBitSize 4012 auto LoS = MIRBuilder.buildShl(HalfTy, InL, Amt); 4013 4014 auto LoOr = MIRBuilder.buildLShr(HalfTy, InL, AmtLack); 4015 auto HiOr = MIRBuilder.buildShl(HalfTy, InH, Amt); 4016 auto HiS = MIRBuilder.buildOr(HalfTy, LoOr, HiOr); 4017 4018 // Long: ShAmt >= NewBitSize 4019 auto LoL = MIRBuilder.buildConstant(HalfTy, 0); // Lo part is zero. 4020 auto HiL = MIRBuilder.buildShl(HalfTy, InL, AmtExcess); // Hi from Lo part. 4021 4022 auto Lo = MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL); 4023 auto Hi = MIRBuilder.buildSelect( 4024 HalfTy, IsZero, InH, MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL)); 4025 4026 ResultRegs[0] = Lo.getReg(0); 4027 ResultRegs[1] = Hi.getReg(0); 4028 break; 4029 } 4030 case TargetOpcode::G_LSHR: 4031 case TargetOpcode::G_ASHR: { 4032 // Short: ShAmt < NewBitSize 4033 auto HiS = MIRBuilder.buildInstr(MI.getOpcode(), {HalfTy}, {InH, Amt}); 4034 4035 auto LoOr = MIRBuilder.buildLShr(HalfTy, InL, Amt); 4036 auto HiOr = MIRBuilder.buildShl(HalfTy, InH, AmtLack); 4037 auto LoS = MIRBuilder.buildOr(HalfTy, LoOr, HiOr); 4038 4039 // Long: ShAmt >= NewBitSize 4040 MachineInstrBuilder HiL; 4041 if (MI.getOpcode() == TargetOpcode::G_LSHR) { 4042 HiL = MIRBuilder.buildConstant(HalfTy, 0); // Hi part is zero. 4043 } else { 4044 auto ShiftAmt = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize - 1); 4045 HiL = MIRBuilder.buildAShr(HalfTy, InH, ShiftAmt); // Sign of Hi part. 4046 } 4047 auto LoL = MIRBuilder.buildInstr(MI.getOpcode(), {HalfTy}, 4048 {InH, AmtExcess}); // Lo from Hi part. 4049 4050 auto Lo = MIRBuilder.buildSelect( 4051 HalfTy, IsZero, InL, MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL)); 4052 4053 auto Hi = MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL); 4054 4055 ResultRegs[0] = Lo.getReg(0); 4056 ResultRegs[1] = Hi.getReg(0); 4057 break; 4058 } 4059 default: 4060 llvm_unreachable("not a shift"); 4061 } 4062 4063 MIRBuilder.buildMerge(DstReg, ResultRegs); 4064 MI.eraseFromParent(); 4065 return Legalized; 4066 } 4067 4068 LegalizerHelper::LegalizeResult 4069 LegalizerHelper::moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx, 4070 LLT MoreTy) { 4071 assert(TypeIdx == 0 && "Expecting only Idx 0"); 4072 4073 Observer.changingInstr(MI); 4074 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) { 4075 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB(); 4076 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator()); 4077 moreElementsVectorSrc(MI, MoreTy, I); 4078 } 4079 4080 MachineBasicBlock &MBB = *MI.getParent(); 4081 MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI()); 4082 moreElementsVectorDst(MI, MoreTy, 0); 4083 Observer.changedInstr(MI); 4084 return Legalized; 4085 } 4086 4087 LegalizerHelper::LegalizeResult 4088 LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx, 4089 LLT MoreTy) { 4090 unsigned Opc = MI.getOpcode(); 4091 switch (Opc) { 4092 case TargetOpcode::G_IMPLICIT_DEF: 4093 case TargetOpcode::G_LOAD: { 4094 if (TypeIdx != 0) 4095 return UnableToLegalize; 4096 Observer.changingInstr(MI); 4097 moreElementsVectorDst(MI, MoreTy, 0); 4098 Observer.changedInstr(MI); 4099 return Legalized; 4100 } 4101 case TargetOpcode::G_STORE: 4102 if (TypeIdx != 0) 4103 return UnableToLegalize; 4104 Observer.changingInstr(MI); 4105 moreElementsVectorSrc(MI, MoreTy, 0); 4106 Observer.changedInstr(MI); 4107 return Legalized; 4108 case TargetOpcode::G_AND: 4109 case TargetOpcode::G_OR: 4110 case TargetOpcode::G_XOR: 4111 case TargetOpcode::G_SMIN: 4112 case TargetOpcode::G_SMAX: 4113 case TargetOpcode::G_UMIN: 4114 case TargetOpcode::G_UMAX: 4115 case TargetOpcode::G_FMINNUM: 4116 case TargetOpcode::G_FMAXNUM: 4117 case TargetOpcode::G_FMINNUM_IEEE: 4118 case TargetOpcode::G_FMAXNUM_IEEE: 4119 case TargetOpcode::G_FMINIMUM: 4120 case TargetOpcode::G_FMAXIMUM: { 4121 Observer.changingInstr(MI); 4122 moreElementsVectorSrc(MI, MoreTy, 1); 4123 moreElementsVectorSrc(MI, MoreTy, 2); 4124 moreElementsVectorDst(MI, MoreTy, 0); 4125 Observer.changedInstr(MI); 4126 return Legalized; 4127 } 4128 case TargetOpcode::G_EXTRACT: 4129 if (TypeIdx != 1) 4130 return UnableToLegalize; 4131 Observer.changingInstr(MI); 4132 moreElementsVectorSrc(MI, MoreTy, 1); 4133 Observer.changedInstr(MI); 4134 return Legalized; 4135 case TargetOpcode::G_INSERT: 4136 case TargetOpcode::G_FREEZE: 4137 if (TypeIdx != 0) 4138 return UnableToLegalize; 4139 Observer.changingInstr(MI); 4140 moreElementsVectorSrc(MI, MoreTy, 1); 4141 moreElementsVectorDst(MI, MoreTy, 0); 4142 Observer.changedInstr(MI); 4143 return Legalized; 4144 case TargetOpcode::G_SELECT: 4145 if (TypeIdx != 0) 4146 return UnableToLegalize; 4147 if (MRI.getType(MI.getOperand(1).getReg()).isVector()) 4148 return UnableToLegalize; 4149 4150 Observer.changingInstr(MI); 4151 moreElementsVectorSrc(MI, MoreTy, 2); 4152 moreElementsVectorSrc(MI, MoreTy, 3); 4153 moreElementsVectorDst(MI, MoreTy, 0); 4154 Observer.changedInstr(MI); 4155 return Legalized; 4156 case TargetOpcode::G_UNMERGE_VALUES: { 4157 if (TypeIdx != 1) 4158 return UnableToLegalize; 4159 4160 LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); 4161 int NumDst = MI.getNumOperands() - 1; 4162 moreElementsVectorSrc(MI, MoreTy, NumDst); 4163 4164 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES); 4165 for (int I = 0; I != NumDst; ++I) 4166 MIB.addDef(MI.getOperand(I).getReg()); 4167 4168 int NewNumDst = MoreTy.getSizeInBits() / DstTy.getSizeInBits(); 4169 for (int I = NumDst; I != NewNumDst; ++I) 4170 MIB.addDef(MRI.createGenericVirtualRegister(DstTy)); 4171 4172 MIB.addUse(MI.getOperand(NumDst).getReg()); 4173 MI.eraseFromParent(); 4174 return Legalized; 4175 } 4176 case TargetOpcode::G_PHI: 4177 return moreElementsVectorPhi(MI, TypeIdx, MoreTy); 4178 default: 4179 return UnableToLegalize; 4180 } 4181 } 4182 4183 void LegalizerHelper::multiplyRegisters(SmallVectorImpl<Register> &DstRegs, 4184 ArrayRef<Register> Src1Regs, 4185 ArrayRef<Register> Src2Regs, 4186 LLT NarrowTy) { 4187 MachineIRBuilder &B = MIRBuilder; 4188 unsigned SrcParts = Src1Regs.size(); 4189 unsigned DstParts = DstRegs.size(); 4190 4191 unsigned DstIdx = 0; // Low bits of the result. 4192 Register FactorSum = 4193 B.buildMul(NarrowTy, Src1Regs[DstIdx], Src2Regs[DstIdx]).getReg(0); 4194 DstRegs[DstIdx] = FactorSum; 4195 4196 unsigned CarrySumPrevDstIdx; 4197 SmallVector<Register, 4> Factors; 4198 4199 for (DstIdx = 1; DstIdx < DstParts; DstIdx++) { 4200 // Collect low parts of muls for DstIdx. 4201 for (unsigned i = DstIdx + 1 < SrcParts ? 0 : DstIdx - SrcParts + 1; 4202 i <= std::min(DstIdx, SrcParts - 1); ++i) { 4203 MachineInstrBuilder Mul = 4204 B.buildMul(NarrowTy, Src1Regs[DstIdx - i], Src2Regs[i]); 4205 Factors.push_back(Mul.getReg(0)); 4206 } 4207 // Collect high parts of muls from previous DstIdx. 4208 for (unsigned i = DstIdx < SrcParts ? 0 : DstIdx - SrcParts; 4209 i <= std::min(DstIdx - 1, SrcParts - 1); ++i) { 4210 MachineInstrBuilder Umulh = 4211 B.buildUMulH(NarrowTy, Src1Regs[DstIdx - 1 - i], Src2Regs[i]); 4212 Factors.push_back(Umulh.getReg(0)); 4213 } 4214 // Add CarrySum from additions calculated for previous DstIdx. 4215 if (DstIdx != 1) { 4216 Factors.push_back(CarrySumPrevDstIdx); 4217 } 4218 4219 Register CarrySum; 4220 // Add all factors and accumulate all carries into CarrySum. 4221 if (DstIdx != DstParts - 1) { 4222 MachineInstrBuilder Uaddo = 4223 B.buildUAddo(NarrowTy, LLT::scalar(1), Factors[0], Factors[1]); 4224 FactorSum = Uaddo.getReg(0); 4225 CarrySum = B.buildZExt(NarrowTy, Uaddo.getReg(1)).getReg(0); 4226 for (unsigned i = 2; i < Factors.size(); ++i) { 4227 MachineInstrBuilder Uaddo = 4228 B.buildUAddo(NarrowTy, LLT::scalar(1), FactorSum, Factors[i]); 4229 FactorSum = Uaddo.getReg(0); 4230 MachineInstrBuilder Carry = B.buildZExt(NarrowTy, Uaddo.getReg(1)); 4231 CarrySum = B.buildAdd(NarrowTy, CarrySum, Carry).getReg(0); 4232 } 4233 } else { 4234 // Since value for the next index is not calculated, neither is CarrySum. 4235 FactorSum = B.buildAdd(NarrowTy, Factors[0], Factors[1]).getReg(0); 4236 for (unsigned i = 2; i < Factors.size(); ++i) 4237 FactorSum = B.buildAdd(NarrowTy, FactorSum, Factors[i]).getReg(0); 4238 } 4239 4240 CarrySumPrevDstIdx = CarrySum; 4241 DstRegs[DstIdx] = FactorSum; 4242 Factors.clear(); 4243 } 4244 } 4245 4246 LegalizerHelper::LegalizeResult 4247 LegalizerHelper::narrowScalarMul(MachineInstr &MI, LLT NarrowTy) { 4248 Register DstReg = MI.getOperand(0).getReg(); 4249 Register Src1 = MI.getOperand(1).getReg(); 4250 Register Src2 = MI.getOperand(2).getReg(); 4251 4252 LLT Ty = MRI.getType(DstReg); 4253 if (Ty.isVector()) 4254 return UnableToLegalize; 4255 4256 unsigned SrcSize = MRI.getType(Src1).getSizeInBits(); 4257 unsigned DstSize = Ty.getSizeInBits(); 4258 unsigned NarrowSize = NarrowTy.getSizeInBits(); 4259 if (DstSize % NarrowSize != 0 || SrcSize % NarrowSize != 0) 4260 return UnableToLegalize; 4261 4262 unsigned NumDstParts = DstSize / NarrowSize; 4263 unsigned NumSrcParts = SrcSize / NarrowSize; 4264 bool IsMulHigh = MI.getOpcode() == TargetOpcode::G_UMULH; 4265 unsigned DstTmpParts = NumDstParts * (IsMulHigh ? 2 : 1); 4266 4267 SmallVector<Register, 2> Src1Parts, Src2Parts; 4268 SmallVector<Register, 2> DstTmpRegs(DstTmpParts); 4269 extractParts(Src1, NarrowTy, NumSrcParts, Src1Parts); 4270 extractParts(Src2, NarrowTy, NumSrcParts, Src2Parts); 4271 multiplyRegisters(DstTmpRegs, Src1Parts, Src2Parts, NarrowTy); 4272 4273 // Take only high half of registers if this is high mul. 4274 ArrayRef<Register> DstRegs( 4275 IsMulHigh ? &DstTmpRegs[DstTmpParts / 2] : &DstTmpRegs[0], NumDstParts); 4276 MIRBuilder.buildMerge(DstReg, DstRegs); 4277 MI.eraseFromParent(); 4278 return Legalized; 4279 } 4280 4281 LegalizerHelper::LegalizeResult 4282 LegalizerHelper::narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx, 4283 LLT NarrowTy) { 4284 if (TypeIdx != 1) 4285 return UnableToLegalize; 4286 4287 uint64_t NarrowSize = NarrowTy.getSizeInBits(); 4288 4289 int64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits(); 4290 // FIXME: add support for when SizeOp1 isn't an exact multiple of 4291 // NarrowSize. 4292 if (SizeOp1 % NarrowSize != 0) 4293 return UnableToLegalize; 4294 int NumParts = SizeOp1 / NarrowSize; 4295 4296 SmallVector<Register, 2> SrcRegs, DstRegs; 4297 SmallVector<uint64_t, 2> Indexes; 4298 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs); 4299 4300 Register OpReg = MI.getOperand(0).getReg(); 4301 uint64_t OpStart = MI.getOperand(2).getImm(); 4302 uint64_t OpSize = MRI.getType(OpReg).getSizeInBits(); 4303 for (int i = 0; i < NumParts; ++i) { 4304 unsigned SrcStart = i * NarrowSize; 4305 4306 if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) { 4307 // No part of the extract uses this subregister, ignore it. 4308 continue; 4309 } else if (SrcStart == OpStart && NarrowTy == MRI.getType(OpReg)) { 4310 // The entire subregister is extracted, forward the value. 4311 DstRegs.push_back(SrcRegs[i]); 4312 continue; 4313 } 4314 4315 // OpSegStart is where this destination segment would start in OpReg if it 4316 // extended infinitely in both directions. 4317 int64_t ExtractOffset; 4318 uint64_t SegSize; 4319 if (OpStart < SrcStart) { 4320 ExtractOffset = 0; 4321 SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart); 4322 } else { 4323 ExtractOffset = OpStart - SrcStart; 4324 SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize); 4325 } 4326 4327 Register SegReg = SrcRegs[i]; 4328 if (ExtractOffset != 0 || SegSize != NarrowSize) { 4329 // A genuine extract is needed. 4330 SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize)); 4331 MIRBuilder.buildExtract(SegReg, SrcRegs[i], ExtractOffset); 4332 } 4333 4334 DstRegs.push_back(SegReg); 4335 } 4336 4337 Register DstReg = MI.getOperand(0).getReg(); 4338 if (MRI.getType(DstReg).isVector()) 4339 MIRBuilder.buildBuildVector(DstReg, DstRegs); 4340 else if (DstRegs.size() > 1) 4341 MIRBuilder.buildMerge(DstReg, DstRegs); 4342 else 4343 MIRBuilder.buildCopy(DstReg, DstRegs[0]); 4344 MI.eraseFromParent(); 4345 return Legalized; 4346 } 4347 4348 LegalizerHelper::LegalizeResult 4349 LegalizerHelper::narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx, 4350 LLT NarrowTy) { 4351 // FIXME: Don't know how to handle secondary types yet. 4352 if (TypeIdx != 0) 4353 return UnableToLegalize; 4354 4355 uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 4356 uint64_t NarrowSize = NarrowTy.getSizeInBits(); 4357 4358 // FIXME: add support for when SizeOp0 isn't an exact multiple of 4359 // NarrowSize. 4360 if (SizeOp0 % NarrowSize != 0) 4361 return UnableToLegalize; 4362 4363 int NumParts = SizeOp0 / NarrowSize; 4364 4365 SmallVector<Register, 2> SrcRegs, DstRegs; 4366 SmallVector<uint64_t, 2> Indexes; 4367 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs); 4368 4369 Register OpReg = MI.getOperand(2).getReg(); 4370 uint64_t OpStart = MI.getOperand(3).getImm(); 4371 uint64_t OpSize = MRI.getType(OpReg).getSizeInBits(); 4372 for (int i = 0; i < NumParts; ++i) { 4373 unsigned DstStart = i * NarrowSize; 4374 4375 if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) { 4376 // No part of the insert affects this subregister, forward the original. 4377 DstRegs.push_back(SrcRegs[i]); 4378 continue; 4379 } else if (DstStart == OpStart && NarrowTy == MRI.getType(OpReg)) { 4380 // The entire subregister is defined by this insert, forward the new 4381 // value. 4382 DstRegs.push_back(OpReg); 4383 continue; 4384 } 4385 4386 // OpSegStart is where this destination segment would start in OpReg if it 4387 // extended infinitely in both directions. 4388 int64_t ExtractOffset, InsertOffset; 4389 uint64_t SegSize; 4390 if (OpStart < DstStart) { 4391 InsertOffset = 0; 4392 ExtractOffset = DstStart - OpStart; 4393 SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart); 4394 } else { 4395 InsertOffset = OpStart - DstStart; 4396 ExtractOffset = 0; 4397 SegSize = 4398 std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart); 4399 } 4400 4401 Register SegReg = OpReg; 4402 if (ExtractOffset != 0 || SegSize != OpSize) { 4403 // A genuine extract is needed. 4404 SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize)); 4405 MIRBuilder.buildExtract(SegReg, OpReg, ExtractOffset); 4406 } 4407 4408 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy); 4409 MIRBuilder.buildInsert(DstReg, SrcRegs[i], SegReg, InsertOffset); 4410 DstRegs.push_back(DstReg); 4411 } 4412 4413 assert(DstRegs.size() == (unsigned)NumParts && "not all parts covered"); 4414 Register DstReg = MI.getOperand(0).getReg(); 4415 if(MRI.getType(DstReg).isVector()) 4416 MIRBuilder.buildBuildVector(DstReg, DstRegs); 4417 else 4418 MIRBuilder.buildMerge(DstReg, DstRegs); 4419 MI.eraseFromParent(); 4420 return Legalized; 4421 } 4422 4423 LegalizerHelper::LegalizeResult 4424 LegalizerHelper::narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx, 4425 LLT NarrowTy) { 4426 Register DstReg = MI.getOperand(0).getReg(); 4427 LLT DstTy = MRI.getType(DstReg); 4428 4429 assert(MI.getNumOperands() == 3 && TypeIdx == 0); 4430 4431 SmallVector<Register, 4> DstRegs, DstLeftoverRegs; 4432 SmallVector<Register, 4> Src0Regs, Src0LeftoverRegs; 4433 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs; 4434 LLT LeftoverTy; 4435 if (!extractParts(MI.getOperand(1).getReg(), DstTy, NarrowTy, LeftoverTy, 4436 Src0Regs, Src0LeftoverRegs)) 4437 return UnableToLegalize; 4438 4439 LLT Unused; 4440 if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, Unused, 4441 Src1Regs, Src1LeftoverRegs)) 4442 llvm_unreachable("inconsistent extractParts result"); 4443 4444 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) { 4445 auto Inst = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy}, 4446 {Src0Regs[I], Src1Regs[I]}); 4447 DstRegs.push_back(Inst.getReg(0)); 4448 } 4449 4450 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) { 4451 auto Inst = MIRBuilder.buildInstr( 4452 MI.getOpcode(), 4453 {LeftoverTy}, {Src0LeftoverRegs[I], Src1LeftoverRegs[I]}); 4454 DstLeftoverRegs.push_back(Inst.getReg(0)); 4455 } 4456 4457 insertParts(DstReg, DstTy, NarrowTy, DstRegs, 4458 LeftoverTy, DstLeftoverRegs); 4459 4460 MI.eraseFromParent(); 4461 return Legalized; 4462 } 4463 4464 LegalizerHelper::LegalizeResult 4465 LegalizerHelper::narrowScalarExt(MachineInstr &MI, unsigned TypeIdx, 4466 LLT NarrowTy) { 4467 if (TypeIdx != 0) 4468 return UnableToLegalize; 4469 4470 Register DstReg = MI.getOperand(0).getReg(); 4471 Register SrcReg = MI.getOperand(1).getReg(); 4472 4473 LLT DstTy = MRI.getType(DstReg); 4474 if (DstTy.isVector()) 4475 return UnableToLegalize; 4476 4477 SmallVector<Register, 8> Parts; 4478 LLT GCDTy = extractGCDType(Parts, DstTy, NarrowTy, SrcReg); 4479 LLT LCMTy = buildLCMMergePieces(DstTy, NarrowTy, GCDTy, Parts, MI.getOpcode()); 4480 buildWidenedRemergeToDst(DstReg, LCMTy, Parts); 4481 4482 MI.eraseFromParent(); 4483 return Legalized; 4484 } 4485 4486 LegalizerHelper::LegalizeResult 4487 LegalizerHelper::narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx, 4488 LLT NarrowTy) { 4489 if (TypeIdx != 0) 4490 return UnableToLegalize; 4491 4492 Register CondReg = MI.getOperand(1).getReg(); 4493 LLT CondTy = MRI.getType(CondReg); 4494 if (CondTy.isVector()) // TODO: Handle vselect 4495 return UnableToLegalize; 4496 4497 Register DstReg = MI.getOperand(0).getReg(); 4498 LLT DstTy = MRI.getType(DstReg); 4499 4500 SmallVector<Register, 4> DstRegs, DstLeftoverRegs; 4501 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs; 4502 SmallVector<Register, 4> Src2Regs, Src2LeftoverRegs; 4503 LLT LeftoverTy; 4504 if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, LeftoverTy, 4505 Src1Regs, Src1LeftoverRegs)) 4506 return UnableToLegalize; 4507 4508 LLT Unused; 4509 if (!extractParts(MI.getOperand(3).getReg(), DstTy, NarrowTy, Unused, 4510 Src2Regs, Src2LeftoverRegs)) 4511 llvm_unreachable("inconsistent extractParts result"); 4512 4513 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) { 4514 auto Select = MIRBuilder.buildSelect(NarrowTy, 4515 CondReg, Src1Regs[I], Src2Regs[I]); 4516 DstRegs.push_back(Select.getReg(0)); 4517 } 4518 4519 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) { 4520 auto Select = MIRBuilder.buildSelect( 4521 LeftoverTy, CondReg, Src1LeftoverRegs[I], Src2LeftoverRegs[I]); 4522 DstLeftoverRegs.push_back(Select.getReg(0)); 4523 } 4524 4525 insertParts(DstReg, DstTy, NarrowTy, DstRegs, 4526 LeftoverTy, DstLeftoverRegs); 4527 4528 MI.eraseFromParent(); 4529 return Legalized; 4530 } 4531 4532 LegalizerHelper::LegalizeResult 4533 LegalizerHelper::narrowScalarCTLZ(MachineInstr &MI, unsigned TypeIdx, 4534 LLT NarrowTy) { 4535 if (TypeIdx != 1) 4536 return UnableToLegalize; 4537 4538 Register DstReg = MI.getOperand(0).getReg(); 4539 Register SrcReg = MI.getOperand(1).getReg(); 4540 LLT DstTy = MRI.getType(DstReg); 4541 LLT SrcTy = MRI.getType(SrcReg); 4542 unsigned NarrowSize = NarrowTy.getSizeInBits(); 4543 4544 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) { 4545 const bool IsUndef = MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF; 4546 4547 MachineIRBuilder &B = MIRBuilder; 4548 auto UnmergeSrc = B.buildUnmerge(NarrowTy, SrcReg); 4549 // ctlz(Hi:Lo) -> Hi == 0 ? (NarrowSize + ctlz(Lo)) : ctlz(Hi) 4550 auto C_0 = B.buildConstant(NarrowTy, 0); 4551 auto HiIsZero = B.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1), 4552 UnmergeSrc.getReg(1), C_0); 4553 auto LoCTLZ = IsUndef ? 4554 B.buildCTLZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(0)) : 4555 B.buildCTLZ(DstTy, UnmergeSrc.getReg(0)); 4556 auto C_NarrowSize = B.buildConstant(DstTy, NarrowSize); 4557 auto HiIsZeroCTLZ = B.buildAdd(DstTy, LoCTLZ, C_NarrowSize); 4558 auto HiCTLZ = B.buildCTLZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(1)); 4559 B.buildSelect(DstReg, HiIsZero, HiIsZeroCTLZ, HiCTLZ); 4560 4561 MI.eraseFromParent(); 4562 return Legalized; 4563 } 4564 4565 return UnableToLegalize; 4566 } 4567 4568 LegalizerHelper::LegalizeResult 4569 LegalizerHelper::narrowScalarCTTZ(MachineInstr &MI, unsigned TypeIdx, 4570 LLT NarrowTy) { 4571 if (TypeIdx != 1) 4572 return UnableToLegalize; 4573 4574 Register DstReg = MI.getOperand(0).getReg(); 4575 Register SrcReg = MI.getOperand(1).getReg(); 4576 LLT DstTy = MRI.getType(DstReg); 4577 LLT SrcTy = MRI.getType(SrcReg); 4578 unsigned NarrowSize = NarrowTy.getSizeInBits(); 4579 4580 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) { 4581 const bool IsUndef = MI.getOpcode() == TargetOpcode::G_CTTZ_ZERO_UNDEF; 4582 4583 MachineIRBuilder &B = MIRBuilder; 4584 auto UnmergeSrc = B.buildUnmerge(NarrowTy, SrcReg); 4585 // cttz(Hi:Lo) -> Lo == 0 ? (cttz(Hi) + NarrowSize) : cttz(Lo) 4586 auto C_0 = B.buildConstant(NarrowTy, 0); 4587 auto LoIsZero = B.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1), 4588 UnmergeSrc.getReg(0), C_0); 4589 auto HiCTTZ = IsUndef ? 4590 B.buildCTTZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(1)) : 4591 B.buildCTTZ(DstTy, UnmergeSrc.getReg(1)); 4592 auto C_NarrowSize = B.buildConstant(DstTy, NarrowSize); 4593 auto LoIsZeroCTTZ = B.buildAdd(DstTy, HiCTTZ, C_NarrowSize); 4594 auto LoCTTZ = B.buildCTTZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(0)); 4595 B.buildSelect(DstReg, LoIsZero, LoIsZeroCTTZ, LoCTTZ); 4596 4597 MI.eraseFromParent(); 4598 return Legalized; 4599 } 4600 4601 return UnableToLegalize; 4602 } 4603 4604 LegalizerHelper::LegalizeResult 4605 LegalizerHelper::narrowScalarCTPOP(MachineInstr &MI, unsigned TypeIdx, 4606 LLT NarrowTy) { 4607 if (TypeIdx != 1) 4608 return UnableToLegalize; 4609 4610 Register DstReg = MI.getOperand(0).getReg(); 4611 LLT DstTy = MRI.getType(DstReg); 4612 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg()); 4613 unsigned NarrowSize = NarrowTy.getSizeInBits(); 4614 4615 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) { 4616 auto UnmergeSrc = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1)); 4617 4618 auto LoCTPOP = MIRBuilder.buildCTPOP(DstTy, UnmergeSrc.getReg(0)); 4619 auto HiCTPOP = MIRBuilder.buildCTPOP(DstTy, UnmergeSrc.getReg(1)); 4620 MIRBuilder.buildAdd(DstReg, HiCTPOP, LoCTPOP); 4621 4622 MI.eraseFromParent(); 4623 return Legalized; 4624 } 4625 4626 return UnableToLegalize; 4627 } 4628 4629 LegalizerHelper::LegalizeResult 4630 LegalizerHelper::lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { 4631 unsigned Opc = MI.getOpcode(); 4632 const auto &TII = MIRBuilder.getTII(); 4633 auto isSupported = [this](const LegalityQuery &Q) { 4634 auto QAction = LI.getAction(Q).Action; 4635 return QAction == Legal || QAction == Libcall || QAction == Custom; 4636 }; 4637 switch (Opc) { 4638 default: 4639 return UnableToLegalize; 4640 case TargetOpcode::G_CTLZ_ZERO_UNDEF: { 4641 // This trivially expands to CTLZ. 4642 Observer.changingInstr(MI); 4643 MI.setDesc(TII.get(TargetOpcode::G_CTLZ)); 4644 Observer.changedInstr(MI); 4645 return Legalized; 4646 } 4647 case TargetOpcode::G_CTLZ: { 4648 Register DstReg = MI.getOperand(0).getReg(); 4649 Register SrcReg = MI.getOperand(1).getReg(); 4650 LLT DstTy = MRI.getType(DstReg); 4651 LLT SrcTy = MRI.getType(SrcReg); 4652 unsigned Len = SrcTy.getSizeInBits(); 4653 4654 if (isSupported({TargetOpcode::G_CTLZ_ZERO_UNDEF, {DstTy, SrcTy}})) { 4655 // If CTLZ_ZERO_UNDEF is supported, emit that and a select for zero. 4656 auto CtlzZU = MIRBuilder.buildCTLZ_ZERO_UNDEF(DstTy, SrcReg); 4657 auto ZeroSrc = MIRBuilder.buildConstant(SrcTy, 0); 4658 auto ICmp = MIRBuilder.buildICmp( 4659 CmpInst::ICMP_EQ, SrcTy.changeElementSize(1), SrcReg, ZeroSrc); 4660 auto LenConst = MIRBuilder.buildConstant(DstTy, Len); 4661 MIRBuilder.buildSelect(DstReg, ICmp, LenConst, CtlzZU); 4662 MI.eraseFromParent(); 4663 return Legalized; 4664 } 4665 // for now, we do this: 4666 // NewLen = NextPowerOf2(Len); 4667 // x = x | (x >> 1); 4668 // x = x | (x >> 2); 4669 // ... 4670 // x = x | (x >>16); 4671 // x = x | (x >>32); // for 64-bit input 4672 // Upto NewLen/2 4673 // return Len - popcount(x); 4674 // 4675 // Ref: "Hacker's Delight" by Henry Warren 4676 Register Op = SrcReg; 4677 unsigned NewLen = PowerOf2Ceil(Len); 4678 for (unsigned i = 0; (1U << i) <= (NewLen / 2); ++i) { 4679 auto MIBShiftAmt = MIRBuilder.buildConstant(SrcTy, 1ULL << i); 4680 auto MIBOp = MIRBuilder.buildOr( 4681 SrcTy, Op, MIRBuilder.buildLShr(SrcTy, Op, MIBShiftAmt)); 4682 Op = MIBOp.getReg(0); 4683 } 4684 auto MIBPop = MIRBuilder.buildCTPOP(DstTy, Op); 4685 MIRBuilder.buildSub(MI.getOperand(0), MIRBuilder.buildConstant(DstTy, Len), 4686 MIBPop); 4687 MI.eraseFromParent(); 4688 return Legalized; 4689 } 4690 case TargetOpcode::G_CTTZ_ZERO_UNDEF: { 4691 // This trivially expands to CTTZ. 4692 Observer.changingInstr(MI); 4693 MI.setDesc(TII.get(TargetOpcode::G_CTTZ)); 4694 Observer.changedInstr(MI); 4695 return Legalized; 4696 } 4697 case TargetOpcode::G_CTTZ: { 4698 Register DstReg = MI.getOperand(0).getReg(); 4699 Register SrcReg = MI.getOperand(1).getReg(); 4700 LLT DstTy = MRI.getType(DstReg); 4701 LLT SrcTy = MRI.getType(SrcReg); 4702 4703 unsigned Len = SrcTy.getSizeInBits(); 4704 if (isSupported({TargetOpcode::G_CTTZ_ZERO_UNDEF, {DstTy, SrcTy}})) { 4705 // If CTTZ_ZERO_UNDEF is legal or custom, emit that and a select with 4706 // zero. 4707 auto CttzZU = MIRBuilder.buildCTTZ_ZERO_UNDEF(DstTy, SrcReg); 4708 auto Zero = MIRBuilder.buildConstant(SrcTy, 0); 4709 auto ICmp = MIRBuilder.buildICmp( 4710 CmpInst::ICMP_EQ, DstTy.changeElementSize(1), SrcReg, Zero); 4711 auto LenConst = MIRBuilder.buildConstant(DstTy, Len); 4712 MIRBuilder.buildSelect(DstReg, ICmp, LenConst, CttzZU); 4713 MI.eraseFromParent(); 4714 return Legalized; 4715 } 4716 // for now, we use: { return popcount(~x & (x - 1)); } 4717 // unless the target has ctlz but not ctpop, in which case we use: 4718 // { return 32 - nlz(~x & (x-1)); } 4719 // Ref: "Hacker's Delight" by Henry Warren 4720 auto MIBCstNeg1 = MIRBuilder.buildConstant(Ty, -1); 4721 auto MIBNot = MIRBuilder.buildXor(Ty, SrcReg, MIBCstNeg1); 4722 auto MIBTmp = MIRBuilder.buildAnd( 4723 Ty, MIBNot, MIRBuilder.buildAdd(Ty, SrcReg, MIBCstNeg1)); 4724 if (!isSupported({TargetOpcode::G_CTPOP, {Ty, Ty}}) && 4725 isSupported({TargetOpcode::G_CTLZ, {Ty, Ty}})) { 4726 auto MIBCstLen = MIRBuilder.buildConstant(Ty, Len); 4727 MIRBuilder.buildSub(MI.getOperand(0), MIBCstLen, 4728 MIRBuilder.buildCTLZ(Ty, MIBTmp)); 4729 MI.eraseFromParent(); 4730 return Legalized; 4731 } 4732 MI.setDesc(TII.get(TargetOpcode::G_CTPOP)); 4733 MI.getOperand(1).setReg(MIBTmp.getReg(0)); 4734 return Legalized; 4735 } 4736 case TargetOpcode::G_CTPOP: { 4737 unsigned Size = Ty.getSizeInBits(); 4738 MachineIRBuilder &B = MIRBuilder; 4739 4740 // Count set bits in blocks of 2 bits. Default approach would be 4741 // B2Count = { val & 0x55555555 } + { (val >> 1) & 0x55555555 } 4742 // We use following formula instead: 4743 // B2Count = val - { (val >> 1) & 0x55555555 } 4744 // since it gives same result in blocks of 2 with one instruction less. 4745 auto C_1 = B.buildConstant(Ty, 1); 4746 auto B2Set1LoTo1Hi = B.buildLShr(Ty, MI.getOperand(1).getReg(), C_1); 4747 APInt B2Mask1HiTo0 = APInt::getSplat(Size, APInt(8, 0x55)); 4748 auto C_B2Mask1HiTo0 = B.buildConstant(Ty, B2Mask1HiTo0); 4749 auto B2Count1Hi = B.buildAnd(Ty, B2Set1LoTo1Hi, C_B2Mask1HiTo0); 4750 auto B2Count = B.buildSub(Ty, MI.getOperand(1).getReg(), B2Count1Hi); 4751 4752 // In order to get count in blocks of 4 add values from adjacent block of 2. 4753 // B4Count = { B2Count & 0x33333333 } + { (B2Count >> 2) & 0x33333333 } 4754 auto C_2 = B.buildConstant(Ty, 2); 4755 auto B4Set2LoTo2Hi = B.buildLShr(Ty, B2Count, C_2); 4756 APInt B4Mask2HiTo0 = APInt::getSplat(Size, APInt(8, 0x33)); 4757 auto C_B4Mask2HiTo0 = B.buildConstant(Ty, B4Mask2HiTo0); 4758 auto B4HiB2Count = B.buildAnd(Ty, B4Set2LoTo2Hi, C_B4Mask2HiTo0); 4759 auto B4LoB2Count = B.buildAnd(Ty, B2Count, C_B4Mask2HiTo0); 4760 auto B4Count = B.buildAdd(Ty, B4HiB2Count, B4LoB2Count); 4761 4762 // For count in blocks of 8 bits we don't have to mask high 4 bits before 4763 // addition since count value sits in range {0,...,8} and 4 bits are enough 4764 // to hold such binary values. After addition high 4 bits still hold count 4765 // of set bits in high 4 bit block, set them to zero and get 8 bit result. 4766 // B8Count = { B4Count + (B4Count >> 4) } & 0x0F0F0F0F 4767 auto C_4 = B.buildConstant(Ty, 4); 4768 auto B8HiB4Count = B.buildLShr(Ty, B4Count, C_4); 4769 auto B8CountDirty4Hi = B.buildAdd(Ty, B8HiB4Count, B4Count); 4770 APInt B8Mask4HiTo0 = APInt::getSplat(Size, APInt(8, 0x0F)); 4771 auto C_B8Mask4HiTo0 = B.buildConstant(Ty, B8Mask4HiTo0); 4772 auto B8Count = B.buildAnd(Ty, B8CountDirty4Hi, C_B8Mask4HiTo0); 4773 4774 assert(Size<=128 && "Scalar size is too large for CTPOP lower algorithm"); 4775 // 8 bits can hold CTPOP result of 128 bit int or smaller. Mul with this 4776 // bitmask will set 8 msb in ResTmp to sum of all B8Counts in 8 bit blocks. 4777 auto MulMask = B.buildConstant(Ty, APInt::getSplat(Size, APInt(8, 0x01))); 4778 auto ResTmp = B.buildMul(Ty, B8Count, MulMask); 4779 4780 // Shift count result from 8 high bits to low bits. 4781 auto C_SizeM8 = B.buildConstant(Ty, Size - 8); 4782 B.buildLShr(MI.getOperand(0).getReg(), ResTmp, C_SizeM8); 4783 4784 MI.eraseFromParent(); 4785 return Legalized; 4786 } 4787 } 4788 } 4789 4790 // Expand s32 = G_UITOFP s64 using bit operations to an IEEE float 4791 // representation. 4792 LegalizerHelper::LegalizeResult 4793 LegalizerHelper::lowerU64ToF32BitOps(MachineInstr &MI) { 4794 Register Dst = MI.getOperand(0).getReg(); 4795 Register Src = MI.getOperand(1).getReg(); 4796 const LLT S64 = LLT::scalar(64); 4797 const LLT S32 = LLT::scalar(32); 4798 const LLT S1 = LLT::scalar(1); 4799 4800 assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S32); 4801 4802 // unsigned cul2f(ulong u) { 4803 // uint lz = clz(u); 4804 // uint e = (u != 0) ? 127U + 63U - lz : 0; 4805 // u = (u << lz) & 0x7fffffffffffffffUL; 4806 // ulong t = u & 0xffffffffffUL; 4807 // uint v = (e << 23) | (uint)(u >> 40); 4808 // uint r = t > 0x8000000000UL ? 1U : (t == 0x8000000000UL ? v & 1U : 0U); 4809 // return as_float(v + r); 4810 // } 4811 4812 auto Zero32 = MIRBuilder.buildConstant(S32, 0); 4813 auto Zero64 = MIRBuilder.buildConstant(S64, 0); 4814 4815 auto LZ = MIRBuilder.buildCTLZ_ZERO_UNDEF(S32, Src); 4816 4817 auto K = MIRBuilder.buildConstant(S32, 127U + 63U); 4818 auto Sub = MIRBuilder.buildSub(S32, K, LZ); 4819 4820 auto NotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, Src, Zero64); 4821 auto E = MIRBuilder.buildSelect(S32, NotZero, Sub, Zero32); 4822 4823 auto Mask0 = MIRBuilder.buildConstant(S64, (-1ULL) >> 1); 4824 auto ShlLZ = MIRBuilder.buildShl(S64, Src, LZ); 4825 4826 auto U = MIRBuilder.buildAnd(S64, ShlLZ, Mask0); 4827 4828 auto Mask1 = MIRBuilder.buildConstant(S64, 0xffffffffffULL); 4829 auto T = MIRBuilder.buildAnd(S64, U, Mask1); 4830 4831 auto UShl = MIRBuilder.buildLShr(S64, U, MIRBuilder.buildConstant(S64, 40)); 4832 auto ShlE = MIRBuilder.buildShl(S32, E, MIRBuilder.buildConstant(S32, 23)); 4833 auto V = MIRBuilder.buildOr(S32, ShlE, MIRBuilder.buildTrunc(S32, UShl)); 4834 4835 auto C = MIRBuilder.buildConstant(S64, 0x8000000000ULL); 4836 auto RCmp = MIRBuilder.buildICmp(CmpInst::ICMP_UGT, S1, T, C); 4837 auto TCmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, T, C); 4838 auto One = MIRBuilder.buildConstant(S32, 1); 4839 4840 auto VTrunc1 = MIRBuilder.buildAnd(S32, V, One); 4841 auto Select0 = MIRBuilder.buildSelect(S32, TCmp, VTrunc1, Zero32); 4842 auto R = MIRBuilder.buildSelect(S32, RCmp, One, Select0); 4843 MIRBuilder.buildAdd(Dst, V, R); 4844 4845 MI.eraseFromParent(); 4846 return Legalized; 4847 } 4848 4849 LegalizerHelper::LegalizeResult 4850 LegalizerHelper::lowerUITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { 4851 Register Dst = MI.getOperand(0).getReg(); 4852 Register Src = MI.getOperand(1).getReg(); 4853 LLT DstTy = MRI.getType(Dst); 4854 LLT SrcTy = MRI.getType(Src); 4855 4856 if (SrcTy == LLT::scalar(1)) { 4857 auto True = MIRBuilder.buildFConstant(DstTy, 1.0); 4858 auto False = MIRBuilder.buildFConstant(DstTy, 0.0); 4859 MIRBuilder.buildSelect(Dst, Src, True, False); 4860 MI.eraseFromParent(); 4861 return Legalized; 4862 } 4863 4864 if (SrcTy != LLT::scalar(64)) 4865 return UnableToLegalize; 4866 4867 if (DstTy == LLT::scalar(32)) { 4868 // TODO: SelectionDAG has several alternative expansions to port which may 4869 // be more reasonble depending on the available instructions. If a target 4870 // has sitofp, does not have CTLZ, or can efficiently use f64 as an 4871 // intermediate type, this is probably worse. 4872 return lowerU64ToF32BitOps(MI); 4873 } 4874 4875 return UnableToLegalize; 4876 } 4877 4878 LegalizerHelper::LegalizeResult 4879 LegalizerHelper::lowerSITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { 4880 Register Dst = MI.getOperand(0).getReg(); 4881 Register Src = MI.getOperand(1).getReg(); 4882 LLT DstTy = MRI.getType(Dst); 4883 LLT SrcTy = MRI.getType(Src); 4884 4885 const LLT S64 = LLT::scalar(64); 4886 const LLT S32 = LLT::scalar(32); 4887 const LLT S1 = LLT::scalar(1); 4888 4889 if (SrcTy == S1) { 4890 auto True = MIRBuilder.buildFConstant(DstTy, -1.0); 4891 auto False = MIRBuilder.buildFConstant(DstTy, 0.0); 4892 MIRBuilder.buildSelect(Dst, Src, True, False); 4893 MI.eraseFromParent(); 4894 return Legalized; 4895 } 4896 4897 if (SrcTy != S64) 4898 return UnableToLegalize; 4899 4900 if (DstTy == S32) { 4901 // signed cl2f(long l) { 4902 // long s = l >> 63; 4903 // float r = cul2f((l + s) ^ s); 4904 // return s ? -r : r; 4905 // } 4906 Register L = Src; 4907 auto SignBit = MIRBuilder.buildConstant(S64, 63); 4908 auto S = MIRBuilder.buildAShr(S64, L, SignBit); 4909 4910 auto LPlusS = MIRBuilder.buildAdd(S64, L, S); 4911 auto Xor = MIRBuilder.buildXor(S64, LPlusS, S); 4912 auto R = MIRBuilder.buildUITOFP(S32, Xor); 4913 4914 auto RNeg = MIRBuilder.buildFNeg(S32, R); 4915 auto SignNotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, S, 4916 MIRBuilder.buildConstant(S64, 0)); 4917 MIRBuilder.buildSelect(Dst, SignNotZero, RNeg, R); 4918 MI.eraseFromParent(); 4919 return Legalized; 4920 } 4921 4922 return UnableToLegalize; 4923 } 4924 4925 LegalizerHelper::LegalizeResult 4926 LegalizerHelper::lowerFPTOUI(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { 4927 Register Dst = MI.getOperand(0).getReg(); 4928 Register Src = MI.getOperand(1).getReg(); 4929 LLT DstTy = MRI.getType(Dst); 4930 LLT SrcTy = MRI.getType(Src); 4931 const LLT S64 = LLT::scalar(64); 4932 const LLT S32 = LLT::scalar(32); 4933 4934 if (SrcTy != S64 && SrcTy != S32) 4935 return UnableToLegalize; 4936 if (DstTy != S32 && DstTy != S64) 4937 return UnableToLegalize; 4938 4939 // FPTOSI gives same result as FPTOUI for positive signed integers. 4940 // FPTOUI needs to deal with fp values that convert to unsigned integers 4941 // greater or equal to 2^31 for float or 2^63 for double. For brevity 2^Exp. 4942 4943 APInt TwoPExpInt = APInt::getSignMask(DstTy.getSizeInBits()); 4944 APFloat TwoPExpFP(SrcTy.getSizeInBits() == 32 ? APFloat::IEEEsingle() 4945 : APFloat::IEEEdouble(), 4946 APInt::getNullValue(SrcTy.getSizeInBits())); 4947 TwoPExpFP.convertFromAPInt(TwoPExpInt, false, APFloat::rmNearestTiesToEven); 4948 4949 MachineInstrBuilder FPTOSI = MIRBuilder.buildFPTOSI(DstTy, Src); 4950 4951 MachineInstrBuilder Threshold = MIRBuilder.buildFConstant(SrcTy, TwoPExpFP); 4952 // For fp Value greater or equal to Threshold(2^Exp), we use FPTOSI on 4953 // (Value - 2^Exp) and add 2^Exp by setting highest bit in result to 1. 4954 MachineInstrBuilder FSub = MIRBuilder.buildFSub(SrcTy, Src, Threshold); 4955 MachineInstrBuilder ResLowBits = MIRBuilder.buildFPTOSI(DstTy, FSub); 4956 MachineInstrBuilder ResHighBit = MIRBuilder.buildConstant(DstTy, TwoPExpInt); 4957 MachineInstrBuilder Res = MIRBuilder.buildXor(DstTy, ResLowBits, ResHighBit); 4958 4959 const LLT S1 = LLT::scalar(1); 4960 4961 MachineInstrBuilder FCMP = 4962 MIRBuilder.buildFCmp(CmpInst::FCMP_ULT, S1, Src, Threshold); 4963 MIRBuilder.buildSelect(Dst, FCMP, FPTOSI, Res); 4964 4965 MI.eraseFromParent(); 4966 return Legalized; 4967 } 4968 4969 LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPTOSI(MachineInstr &MI) { 4970 Register Dst = MI.getOperand(0).getReg(); 4971 Register Src = MI.getOperand(1).getReg(); 4972 LLT DstTy = MRI.getType(Dst); 4973 LLT SrcTy = MRI.getType(Src); 4974 const LLT S64 = LLT::scalar(64); 4975 const LLT S32 = LLT::scalar(32); 4976 4977 // FIXME: Only f32 to i64 conversions are supported. 4978 if (SrcTy.getScalarType() != S32 || DstTy.getScalarType() != S64) 4979 return UnableToLegalize; 4980 4981 // Expand f32 -> i64 conversion 4982 // This algorithm comes from compiler-rt's implementation of fixsfdi: 4983 // https://github.com/llvm/llvm-project/blob/master/compiler-rt/lib/builtins/fixsfdi.c 4984 4985 unsigned SrcEltBits = SrcTy.getScalarSizeInBits(); 4986 4987 auto ExponentMask = MIRBuilder.buildConstant(SrcTy, 0x7F800000); 4988 auto ExponentLoBit = MIRBuilder.buildConstant(SrcTy, 23); 4989 4990 auto AndExpMask = MIRBuilder.buildAnd(SrcTy, Src, ExponentMask); 4991 auto ExponentBits = MIRBuilder.buildLShr(SrcTy, AndExpMask, ExponentLoBit); 4992 4993 auto SignMask = MIRBuilder.buildConstant(SrcTy, 4994 APInt::getSignMask(SrcEltBits)); 4995 auto AndSignMask = MIRBuilder.buildAnd(SrcTy, Src, SignMask); 4996 auto SignLowBit = MIRBuilder.buildConstant(SrcTy, SrcEltBits - 1); 4997 auto Sign = MIRBuilder.buildAShr(SrcTy, AndSignMask, SignLowBit); 4998 Sign = MIRBuilder.buildSExt(DstTy, Sign); 4999 5000 auto MantissaMask = MIRBuilder.buildConstant(SrcTy, 0x007FFFFF); 5001 auto AndMantissaMask = MIRBuilder.buildAnd(SrcTy, Src, MantissaMask); 5002 auto K = MIRBuilder.buildConstant(SrcTy, 0x00800000); 5003 5004 auto R = MIRBuilder.buildOr(SrcTy, AndMantissaMask, K); 5005 R = MIRBuilder.buildZExt(DstTy, R); 5006 5007 auto Bias = MIRBuilder.buildConstant(SrcTy, 127); 5008 auto Exponent = MIRBuilder.buildSub(SrcTy, ExponentBits, Bias); 5009 auto SubExponent = MIRBuilder.buildSub(SrcTy, Exponent, ExponentLoBit); 5010 auto ExponentSub = MIRBuilder.buildSub(SrcTy, ExponentLoBit, Exponent); 5011 5012 auto Shl = MIRBuilder.buildShl(DstTy, R, SubExponent); 5013 auto Srl = MIRBuilder.buildLShr(DstTy, R, ExponentSub); 5014 5015 const LLT S1 = LLT::scalar(1); 5016 auto CmpGt = MIRBuilder.buildICmp(CmpInst::ICMP_SGT, 5017 S1, Exponent, ExponentLoBit); 5018 5019 R = MIRBuilder.buildSelect(DstTy, CmpGt, Shl, Srl); 5020 5021 auto XorSign = MIRBuilder.buildXor(DstTy, R, Sign); 5022 auto Ret = MIRBuilder.buildSub(DstTy, XorSign, Sign); 5023 5024 auto ZeroSrcTy = MIRBuilder.buildConstant(SrcTy, 0); 5025 5026 auto ExponentLt0 = MIRBuilder.buildICmp(CmpInst::ICMP_SLT, 5027 S1, Exponent, ZeroSrcTy); 5028 5029 auto ZeroDstTy = MIRBuilder.buildConstant(DstTy, 0); 5030 MIRBuilder.buildSelect(Dst, ExponentLt0, ZeroDstTy, Ret); 5031 5032 MI.eraseFromParent(); 5033 return Legalized; 5034 } 5035 5036 // f64 -> f16 conversion using round-to-nearest-even rounding mode. 5037 LegalizerHelper::LegalizeResult 5038 LegalizerHelper::lowerFPTRUNC_F64_TO_F16(MachineInstr &MI) { 5039 Register Dst = MI.getOperand(0).getReg(); 5040 Register Src = MI.getOperand(1).getReg(); 5041 5042 if (MRI.getType(Src).isVector()) // TODO: Handle vectors directly. 5043 return UnableToLegalize; 5044 5045 const unsigned ExpMask = 0x7ff; 5046 const unsigned ExpBiasf64 = 1023; 5047 const unsigned ExpBiasf16 = 15; 5048 const LLT S32 = LLT::scalar(32); 5049 const LLT S1 = LLT::scalar(1); 5050 5051 auto Unmerge = MIRBuilder.buildUnmerge(S32, Src); 5052 Register U = Unmerge.getReg(0); 5053 Register UH = Unmerge.getReg(1); 5054 5055 auto E = MIRBuilder.buildLShr(S32, UH, MIRBuilder.buildConstant(S32, 20)); 5056 E = MIRBuilder.buildAnd(S32, E, MIRBuilder.buildConstant(S32, ExpMask)); 5057 5058 // Subtract the fp64 exponent bias (1023) to get the real exponent and 5059 // add the f16 bias (15) to get the biased exponent for the f16 format. 5060 E = MIRBuilder.buildAdd( 5061 S32, E, MIRBuilder.buildConstant(S32, -ExpBiasf64 + ExpBiasf16)); 5062 5063 auto M = MIRBuilder.buildLShr(S32, UH, MIRBuilder.buildConstant(S32, 8)); 5064 M = MIRBuilder.buildAnd(S32, M, MIRBuilder.buildConstant(S32, 0xffe)); 5065 5066 auto MaskedSig = MIRBuilder.buildAnd(S32, UH, 5067 MIRBuilder.buildConstant(S32, 0x1ff)); 5068 MaskedSig = MIRBuilder.buildOr(S32, MaskedSig, U); 5069 5070 auto Zero = MIRBuilder.buildConstant(S32, 0); 5071 auto SigCmpNE0 = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, MaskedSig, Zero); 5072 auto Lo40Set = MIRBuilder.buildZExt(S32, SigCmpNE0); 5073 M = MIRBuilder.buildOr(S32, M, Lo40Set); 5074 5075 // (M != 0 ? 0x0200 : 0) | 0x7c00; 5076 auto Bits0x200 = MIRBuilder.buildConstant(S32, 0x0200); 5077 auto CmpM_NE0 = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, M, Zero); 5078 auto SelectCC = MIRBuilder.buildSelect(S32, CmpM_NE0, Bits0x200, Zero); 5079 5080 auto Bits0x7c00 = MIRBuilder.buildConstant(S32, 0x7c00); 5081 auto I = MIRBuilder.buildOr(S32, SelectCC, Bits0x7c00); 5082 5083 // N = M | (E << 12); 5084 auto EShl12 = MIRBuilder.buildShl(S32, E, MIRBuilder.buildConstant(S32, 12)); 5085 auto N = MIRBuilder.buildOr(S32, M, EShl12); 5086 5087 // B = clamp(1-E, 0, 13); 5088 auto One = MIRBuilder.buildConstant(S32, 1); 5089 auto OneSubExp = MIRBuilder.buildSub(S32, One, E); 5090 auto B = MIRBuilder.buildSMax(S32, OneSubExp, Zero); 5091 B = MIRBuilder.buildSMin(S32, B, MIRBuilder.buildConstant(S32, 13)); 5092 5093 auto SigSetHigh = MIRBuilder.buildOr(S32, M, 5094 MIRBuilder.buildConstant(S32, 0x1000)); 5095 5096 auto D = MIRBuilder.buildLShr(S32, SigSetHigh, B); 5097 auto D0 = MIRBuilder.buildShl(S32, D, B); 5098 5099 auto D0_NE_SigSetHigh = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, 5100 D0, SigSetHigh); 5101 auto D1 = MIRBuilder.buildZExt(S32, D0_NE_SigSetHigh); 5102 D = MIRBuilder.buildOr(S32, D, D1); 5103 5104 auto CmpELtOne = MIRBuilder.buildICmp(CmpInst::ICMP_SLT, S1, E, One); 5105 auto V = MIRBuilder.buildSelect(S32, CmpELtOne, D, N); 5106 5107 auto VLow3 = MIRBuilder.buildAnd(S32, V, MIRBuilder.buildConstant(S32, 7)); 5108 V = MIRBuilder.buildLShr(S32, V, MIRBuilder.buildConstant(S32, 2)); 5109 5110 auto VLow3Eq3 = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, VLow3, 5111 MIRBuilder.buildConstant(S32, 3)); 5112 auto V0 = MIRBuilder.buildZExt(S32, VLow3Eq3); 5113 5114 auto VLow3Gt5 = MIRBuilder.buildICmp(CmpInst::ICMP_SGT, S1, VLow3, 5115 MIRBuilder.buildConstant(S32, 5)); 5116 auto V1 = MIRBuilder.buildZExt(S32, VLow3Gt5); 5117 5118 V1 = MIRBuilder.buildOr(S32, V0, V1); 5119 V = MIRBuilder.buildAdd(S32, V, V1); 5120 5121 auto CmpEGt30 = MIRBuilder.buildICmp(CmpInst::ICMP_SGT, S1, 5122 E, MIRBuilder.buildConstant(S32, 30)); 5123 V = MIRBuilder.buildSelect(S32, CmpEGt30, 5124 MIRBuilder.buildConstant(S32, 0x7c00), V); 5125 5126 auto CmpEGt1039 = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, 5127 E, MIRBuilder.buildConstant(S32, 1039)); 5128 V = MIRBuilder.buildSelect(S32, CmpEGt1039, I, V); 5129 5130 // Extract the sign bit. 5131 auto Sign = MIRBuilder.buildLShr(S32, UH, MIRBuilder.buildConstant(S32, 16)); 5132 Sign = MIRBuilder.buildAnd(S32, Sign, MIRBuilder.buildConstant(S32, 0x8000)); 5133 5134 // Insert the sign bit 5135 V = MIRBuilder.buildOr(S32, Sign, V); 5136 5137 MIRBuilder.buildTrunc(Dst, V); 5138 MI.eraseFromParent(); 5139 return Legalized; 5140 } 5141 5142 LegalizerHelper::LegalizeResult 5143 LegalizerHelper::lowerFPTRUNC(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { 5144 Register Dst = MI.getOperand(0).getReg(); 5145 Register Src = MI.getOperand(1).getReg(); 5146 5147 LLT DstTy = MRI.getType(Dst); 5148 LLT SrcTy = MRI.getType(Src); 5149 const LLT S64 = LLT::scalar(64); 5150 const LLT S16 = LLT::scalar(16); 5151 5152 if (DstTy.getScalarType() == S16 && SrcTy.getScalarType() == S64) 5153 return lowerFPTRUNC_F64_TO_F16(MI); 5154 5155 return UnableToLegalize; 5156 } 5157 5158 // TODO: If RHS is a constant SelectionDAGBuilder expands this into a 5159 // multiplication tree. 5160 LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPOWI(MachineInstr &MI) { 5161 Register Dst = MI.getOperand(0).getReg(); 5162 Register Src0 = MI.getOperand(1).getReg(); 5163 Register Src1 = MI.getOperand(2).getReg(); 5164 LLT Ty = MRI.getType(Dst); 5165 5166 auto CvtSrc1 = MIRBuilder.buildSITOFP(Ty, Src1); 5167 MIRBuilder.buildFPow(Dst, Src0, CvtSrc1, MI.getFlags()); 5168 MI.eraseFromParent(); 5169 return Legalized; 5170 } 5171 5172 static CmpInst::Predicate minMaxToCompare(unsigned Opc) { 5173 switch (Opc) { 5174 case TargetOpcode::G_SMIN: 5175 return CmpInst::ICMP_SLT; 5176 case TargetOpcode::G_SMAX: 5177 return CmpInst::ICMP_SGT; 5178 case TargetOpcode::G_UMIN: 5179 return CmpInst::ICMP_ULT; 5180 case TargetOpcode::G_UMAX: 5181 return CmpInst::ICMP_UGT; 5182 default: 5183 llvm_unreachable("not in integer min/max"); 5184 } 5185 } 5186 5187 LegalizerHelper::LegalizeResult 5188 LegalizerHelper::lowerMinMax(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { 5189 Register Dst = MI.getOperand(0).getReg(); 5190 Register Src0 = MI.getOperand(1).getReg(); 5191 Register Src1 = MI.getOperand(2).getReg(); 5192 5193 const CmpInst::Predicate Pred = minMaxToCompare(MI.getOpcode()); 5194 LLT CmpType = MRI.getType(Dst).changeElementSize(1); 5195 5196 auto Cmp = MIRBuilder.buildICmp(Pred, CmpType, Src0, Src1); 5197 MIRBuilder.buildSelect(Dst, Cmp, Src0, Src1); 5198 5199 MI.eraseFromParent(); 5200 return Legalized; 5201 } 5202 5203 LegalizerHelper::LegalizeResult 5204 LegalizerHelper::lowerFCopySign(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { 5205 Register Dst = MI.getOperand(0).getReg(); 5206 Register Src0 = MI.getOperand(1).getReg(); 5207 Register Src1 = MI.getOperand(2).getReg(); 5208 5209 const LLT Src0Ty = MRI.getType(Src0); 5210 const LLT Src1Ty = MRI.getType(Src1); 5211 5212 const int Src0Size = Src0Ty.getScalarSizeInBits(); 5213 const int Src1Size = Src1Ty.getScalarSizeInBits(); 5214 5215 auto SignBitMask = MIRBuilder.buildConstant( 5216 Src0Ty, APInt::getSignMask(Src0Size)); 5217 5218 auto NotSignBitMask = MIRBuilder.buildConstant( 5219 Src0Ty, APInt::getLowBitsSet(Src0Size, Src0Size - 1)); 5220 5221 auto And0 = MIRBuilder.buildAnd(Src0Ty, Src0, NotSignBitMask); 5222 MachineInstr *Or; 5223 5224 if (Src0Ty == Src1Ty) { 5225 auto And1 = MIRBuilder.buildAnd(Src1Ty, Src1, SignBitMask); 5226 Or = MIRBuilder.buildOr(Dst, And0, And1); 5227 } else if (Src0Size > Src1Size) { 5228 auto ShiftAmt = MIRBuilder.buildConstant(Src0Ty, Src0Size - Src1Size); 5229 auto Zext = MIRBuilder.buildZExt(Src0Ty, Src1); 5230 auto Shift = MIRBuilder.buildShl(Src0Ty, Zext, ShiftAmt); 5231 auto And1 = MIRBuilder.buildAnd(Src0Ty, Shift, SignBitMask); 5232 Or = MIRBuilder.buildOr(Dst, And0, And1); 5233 } else { 5234 auto ShiftAmt = MIRBuilder.buildConstant(Src1Ty, Src1Size - Src0Size); 5235 auto Shift = MIRBuilder.buildLShr(Src1Ty, Src1, ShiftAmt); 5236 auto Trunc = MIRBuilder.buildTrunc(Src0Ty, Shift); 5237 auto And1 = MIRBuilder.buildAnd(Src0Ty, Trunc, SignBitMask); 5238 Or = MIRBuilder.buildOr(Dst, And0, And1); 5239 } 5240 5241 // Be careful about setting nsz/nnan/ninf on every instruction, since the 5242 // constants are a nan and -0.0, but the final result should preserve 5243 // everything. 5244 if (unsigned Flags = MI.getFlags()) 5245 Or->setFlags(Flags); 5246 5247 MI.eraseFromParent(); 5248 return Legalized; 5249 } 5250 5251 LegalizerHelper::LegalizeResult 5252 LegalizerHelper::lowerFMinNumMaxNum(MachineInstr &MI) { 5253 unsigned NewOp = MI.getOpcode() == TargetOpcode::G_FMINNUM ? 5254 TargetOpcode::G_FMINNUM_IEEE : TargetOpcode::G_FMAXNUM_IEEE; 5255 5256 Register Dst = MI.getOperand(0).getReg(); 5257 Register Src0 = MI.getOperand(1).getReg(); 5258 Register Src1 = MI.getOperand(2).getReg(); 5259 LLT Ty = MRI.getType(Dst); 5260 5261 if (!MI.getFlag(MachineInstr::FmNoNans)) { 5262 // Insert canonicalizes if it's possible we need to quiet to get correct 5263 // sNaN behavior. 5264 5265 // Note this must be done here, and not as an optimization combine in the 5266 // absence of a dedicate quiet-snan instruction as we're using an 5267 // omni-purpose G_FCANONICALIZE. 5268 if (!isKnownNeverSNaN(Src0, MRI)) 5269 Src0 = MIRBuilder.buildFCanonicalize(Ty, Src0, MI.getFlags()).getReg(0); 5270 5271 if (!isKnownNeverSNaN(Src1, MRI)) 5272 Src1 = MIRBuilder.buildFCanonicalize(Ty, Src1, MI.getFlags()).getReg(0); 5273 } 5274 5275 // If there are no nans, it's safe to simply replace this with the non-IEEE 5276 // version. 5277 MIRBuilder.buildInstr(NewOp, {Dst}, {Src0, Src1}, MI.getFlags()); 5278 MI.eraseFromParent(); 5279 return Legalized; 5280 } 5281 5282 LegalizerHelper::LegalizeResult LegalizerHelper::lowerFMad(MachineInstr &MI) { 5283 // Expand G_FMAD a, b, c -> G_FADD (G_FMUL a, b), c 5284 Register DstReg = MI.getOperand(0).getReg(); 5285 LLT Ty = MRI.getType(DstReg); 5286 unsigned Flags = MI.getFlags(); 5287 5288 auto Mul = MIRBuilder.buildFMul(Ty, MI.getOperand(1), MI.getOperand(2), 5289 Flags); 5290 MIRBuilder.buildFAdd(DstReg, Mul, MI.getOperand(3), Flags); 5291 MI.eraseFromParent(); 5292 return Legalized; 5293 } 5294 5295 LegalizerHelper::LegalizeResult 5296 LegalizerHelper::lowerIntrinsicRound(MachineInstr &MI) { 5297 Register DstReg = MI.getOperand(0).getReg(); 5298 Register X = MI.getOperand(1).getReg(); 5299 const unsigned Flags = MI.getFlags(); 5300 const LLT Ty = MRI.getType(DstReg); 5301 const LLT CondTy = Ty.changeElementSize(1); 5302 5303 // round(x) => 5304 // t = trunc(x); 5305 // d = fabs(x - t); 5306 // o = copysign(1.0f, x); 5307 // return t + (d >= 0.5 ? o : 0.0); 5308 5309 auto T = MIRBuilder.buildIntrinsicTrunc(Ty, X, Flags); 5310 5311 auto Diff = MIRBuilder.buildFSub(Ty, X, T, Flags); 5312 auto AbsDiff = MIRBuilder.buildFAbs(Ty, Diff, Flags); 5313 auto Zero = MIRBuilder.buildFConstant(Ty, 0.0); 5314 auto One = MIRBuilder.buildFConstant(Ty, 1.0); 5315 auto Half = MIRBuilder.buildFConstant(Ty, 0.5); 5316 auto SignOne = MIRBuilder.buildFCopysign(Ty, One, X); 5317 5318 auto Cmp = MIRBuilder.buildFCmp(CmpInst::FCMP_OGE, CondTy, AbsDiff, Half, 5319 Flags); 5320 auto Sel = MIRBuilder.buildSelect(Ty, Cmp, SignOne, Zero, Flags); 5321 5322 MIRBuilder.buildFAdd(DstReg, T, Sel, Flags); 5323 5324 MI.eraseFromParent(); 5325 return Legalized; 5326 } 5327 5328 LegalizerHelper::LegalizeResult 5329 LegalizerHelper::lowerFFloor(MachineInstr &MI) { 5330 Register DstReg = MI.getOperand(0).getReg(); 5331 Register SrcReg = MI.getOperand(1).getReg(); 5332 unsigned Flags = MI.getFlags(); 5333 LLT Ty = MRI.getType(DstReg); 5334 const LLT CondTy = Ty.changeElementSize(1); 5335 5336 // result = trunc(src); 5337 // if (src < 0.0 && src != result) 5338 // result += -1.0. 5339 5340 auto Trunc = MIRBuilder.buildIntrinsicTrunc(Ty, SrcReg, Flags); 5341 auto Zero = MIRBuilder.buildFConstant(Ty, 0.0); 5342 5343 auto Lt0 = MIRBuilder.buildFCmp(CmpInst::FCMP_OLT, CondTy, 5344 SrcReg, Zero, Flags); 5345 auto NeTrunc = MIRBuilder.buildFCmp(CmpInst::FCMP_ONE, CondTy, 5346 SrcReg, Trunc, Flags); 5347 auto And = MIRBuilder.buildAnd(CondTy, Lt0, NeTrunc); 5348 auto AddVal = MIRBuilder.buildSITOFP(Ty, And); 5349 5350 MIRBuilder.buildFAdd(DstReg, Trunc, AddVal, Flags); 5351 MI.eraseFromParent(); 5352 return Legalized; 5353 } 5354 5355 LegalizerHelper::LegalizeResult 5356 LegalizerHelper::lowerMergeValues(MachineInstr &MI) { 5357 const unsigned NumOps = MI.getNumOperands(); 5358 Register DstReg = MI.getOperand(0).getReg(); 5359 Register Src0Reg = MI.getOperand(1).getReg(); 5360 LLT DstTy = MRI.getType(DstReg); 5361 LLT SrcTy = MRI.getType(Src0Reg); 5362 unsigned PartSize = SrcTy.getSizeInBits(); 5363 5364 LLT WideTy = LLT::scalar(DstTy.getSizeInBits()); 5365 Register ResultReg = MIRBuilder.buildZExt(WideTy, Src0Reg).getReg(0); 5366 5367 for (unsigned I = 2; I != NumOps; ++I) { 5368 const unsigned Offset = (I - 1) * PartSize; 5369 5370 Register SrcReg = MI.getOperand(I).getReg(); 5371 auto ZextInput = MIRBuilder.buildZExt(WideTy, SrcReg); 5372 5373 Register NextResult = I + 1 == NumOps && WideTy == DstTy ? DstReg : 5374 MRI.createGenericVirtualRegister(WideTy); 5375 5376 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, Offset); 5377 auto Shl = MIRBuilder.buildShl(WideTy, ZextInput, ShiftAmt); 5378 MIRBuilder.buildOr(NextResult, ResultReg, Shl); 5379 ResultReg = NextResult; 5380 } 5381 5382 if (DstTy.isPointer()) { 5383 if (MIRBuilder.getDataLayout().isNonIntegralAddressSpace( 5384 DstTy.getAddressSpace())) { 5385 LLVM_DEBUG(dbgs() << "Not casting nonintegral address space\n"); 5386 return UnableToLegalize; 5387 } 5388 5389 MIRBuilder.buildIntToPtr(DstReg, ResultReg); 5390 } 5391 5392 MI.eraseFromParent(); 5393 return Legalized; 5394 } 5395 5396 LegalizerHelper::LegalizeResult 5397 LegalizerHelper::lowerUnmergeValues(MachineInstr &MI) { 5398 const unsigned NumDst = MI.getNumOperands() - 1; 5399 Register SrcReg = MI.getOperand(NumDst).getReg(); 5400 Register Dst0Reg = MI.getOperand(0).getReg(); 5401 LLT DstTy = MRI.getType(Dst0Reg); 5402 if (DstTy.isPointer()) 5403 return UnableToLegalize; // TODO 5404 5405 SrcReg = coerceToScalar(SrcReg); 5406 if (!SrcReg) 5407 return UnableToLegalize; 5408 5409 // Expand scalarizing unmerge as bitcast to integer and shift. 5410 LLT IntTy = MRI.getType(SrcReg); 5411 5412 MIRBuilder.buildTrunc(Dst0Reg, SrcReg); 5413 5414 const unsigned DstSize = DstTy.getSizeInBits(); 5415 unsigned Offset = DstSize; 5416 for (unsigned I = 1; I != NumDst; ++I, Offset += DstSize) { 5417 auto ShiftAmt = MIRBuilder.buildConstant(IntTy, Offset); 5418 auto Shift = MIRBuilder.buildLShr(IntTy, SrcReg, ShiftAmt); 5419 MIRBuilder.buildTrunc(MI.getOperand(I), Shift); 5420 } 5421 5422 MI.eraseFromParent(); 5423 return Legalized; 5424 } 5425 5426 /// Lower a vector extract by writing the vector to a stack temporary and 5427 /// reloading the element. 5428 /// 5429 /// %dst = G_EXTRACT_VECTOR_ELT %vec, %idx 5430 /// => 5431 /// %stack_temp = G_FRAME_INDEX 5432 /// G_STORE %vec, %stack_temp 5433 /// %idx = clamp(%idx, %vec.getNumElements()) 5434 /// %element_ptr = G_PTR_ADD %stack_temp, %idx 5435 /// %dst = G_LOAD %element_ptr 5436 LegalizerHelper::LegalizeResult 5437 LegalizerHelper::lowerExtractVectorElt(MachineInstr &MI) { 5438 Register DstReg = MI.getOperand(0).getReg(); 5439 Register SrcVec = MI.getOperand(1).getReg(); 5440 Register Idx = MI.getOperand(2).getReg(); 5441 LLT VecTy = MRI.getType(SrcVec); 5442 LLT EltTy = VecTy.getElementType(); 5443 if (!EltTy.isByteSized()) { // Not implemented. 5444 LLVM_DEBUG(dbgs() << "Can't handle non-byte element vectors yet\n"); 5445 return UnableToLegalize; 5446 } 5447 5448 unsigned EltBytes = EltTy.getSizeInBytes(); 5449 Align StoreAlign = getStackTemporaryAlignment(VecTy); 5450 Align LoadAlign; 5451 5452 MachinePointerInfo PtrInfo; 5453 auto StackTemp = createStackTemporary(TypeSize::Fixed(VecTy.getSizeInBytes()), 5454 StoreAlign, PtrInfo); 5455 MIRBuilder.buildStore(SrcVec, StackTemp, PtrInfo, StoreAlign); 5456 5457 // Get the pointer to the element, and be sure not to hit undefined behavior 5458 // if the index is out of bounds. 5459 Register LoadPtr = getVectorElementPointer(StackTemp.getReg(0), VecTy, Idx); 5460 5461 int64_t IdxVal; 5462 if (mi_match(Idx, MRI, m_ICst(IdxVal))) { 5463 int64_t Offset = IdxVal * EltBytes; 5464 PtrInfo = PtrInfo.getWithOffset(Offset); 5465 LoadAlign = commonAlignment(StoreAlign, Offset); 5466 } else { 5467 // We lose information with a variable offset. 5468 LoadAlign = getStackTemporaryAlignment(EltTy); 5469 PtrInfo = MachinePointerInfo(MRI.getType(LoadPtr).getAddressSpace()); 5470 } 5471 5472 MIRBuilder.buildLoad(DstReg, LoadPtr, PtrInfo, LoadAlign); 5473 MI.eraseFromParent(); 5474 return Legalized; 5475 } 5476 5477 LegalizerHelper::LegalizeResult 5478 LegalizerHelper::lowerShuffleVector(MachineInstr &MI) { 5479 Register DstReg = MI.getOperand(0).getReg(); 5480 Register Src0Reg = MI.getOperand(1).getReg(); 5481 Register Src1Reg = MI.getOperand(2).getReg(); 5482 LLT Src0Ty = MRI.getType(Src0Reg); 5483 LLT DstTy = MRI.getType(DstReg); 5484 LLT IdxTy = LLT::scalar(32); 5485 5486 ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask(); 5487 5488 if (DstTy.isScalar()) { 5489 if (Src0Ty.isVector()) 5490 return UnableToLegalize; 5491 5492 // This is just a SELECT. 5493 assert(Mask.size() == 1 && "Expected a single mask element"); 5494 Register Val; 5495 if (Mask[0] < 0 || Mask[0] > 1) 5496 Val = MIRBuilder.buildUndef(DstTy).getReg(0); 5497 else 5498 Val = Mask[0] == 0 ? Src0Reg : Src1Reg; 5499 MIRBuilder.buildCopy(DstReg, Val); 5500 MI.eraseFromParent(); 5501 return Legalized; 5502 } 5503 5504 Register Undef; 5505 SmallVector<Register, 32> BuildVec; 5506 LLT EltTy = DstTy.getElementType(); 5507 5508 for (int Idx : Mask) { 5509 if (Idx < 0) { 5510 if (!Undef.isValid()) 5511 Undef = MIRBuilder.buildUndef(EltTy).getReg(0); 5512 BuildVec.push_back(Undef); 5513 continue; 5514 } 5515 5516 if (Src0Ty.isScalar()) { 5517 BuildVec.push_back(Idx == 0 ? Src0Reg : Src1Reg); 5518 } else { 5519 int NumElts = Src0Ty.getNumElements(); 5520 Register SrcVec = Idx < NumElts ? Src0Reg : Src1Reg; 5521 int ExtractIdx = Idx < NumElts ? Idx : Idx - NumElts; 5522 auto IdxK = MIRBuilder.buildConstant(IdxTy, ExtractIdx); 5523 auto Extract = MIRBuilder.buildExtractVectorElement(EltTy, SrcVec, IdxK); 5524 BuildVec.push_back(Extract.getReg(0)); 5525 } 5526 } 5527 5528 MIRBuilder.buildBuildVector(DstReg, BuildVec); 5529 MI.eraseFromParent(); 5530 return Legalized; 5531 } 5532 5533 LegalizerHelper::LegalizeResult 5534 LegalizerHelper::lowerDynStackAlloc(MachineInstr &MI) { 5535 const auto &MF = *MI.getMF(); 5536 const auto &TFI = *MF.getSubtarget().getFrameLowering(); 5537 if (TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp) 5538 return UnableToLegalize; 5539 5540 Register Dst = MI.getOperand(0).getReg(); 5541 Register AllocSize = MI.getOperand(1).getReg(); 5542 Align Alignment = assumeAligned(MI.getOperand(2).getImm()); 5543 5544 LLT PtrTy = MRI.getType(Dst); 5545 LLT IntPtrTy = LLT::scalar(PtrTy.getSizeInBits()); 5546 5547 const auto &TLI = *MF.getSubtarget().getTargetLowering(); 5548 Register SPReg = TLI.getStackPointerRegisterToSaveRestore(); 5549 auto SPTmp = MIRBuilder.buildCopy(PtrTy, SPReg); 5550 SPTmp = MIRBuilder.buildCast(IntPtrTy, SPTmp); 5551 5552 // Subtract the final alloc from the SP. We use G_PTRTOINT here so we don't 5553 // have to generate an extra instruction to negate the alloc and then use 5554 // G_PTR_ADD to add the negative offset. 5555 auto Alloc = MIRBuilder.buildSub(IntPtrTy, SPTmp, AllocSize); 5556 if (Alignment > Align(1)) { 5557 APInt AlignMask(IntPtrTy.getSizeInBits(), Alignment.value(), true); 5558 AlignMask.negate(); 5559 auto AlignCst = MIRBuilder.buildConstant(IntPtrTy, AlignMask); 5560 Alloc = MIRBuilder.buildAnd(IntPtrTy, Alloc, AlignCst); 5561 } 5562 5563 SPTmp = MIRBuilder.buildCast(PtrTy, Alloc); 5564 MIRBuilder.buildCopy(SPReg, SPTmp); 5565 MIRBuilder.buildCopy(Dst, SPTmp); 5566 5567 MI.eraseFromParent(); 5568 return Legalized; 5569 } 5570 5571 LegalizerHelper::LegalizeResult 5572 LegalizerHelper::lowerExtract(MachineInstr &MI) { 5573 Register Dst = MI.getOperand(0).getReg(); 5574 Register Src = MI.getOperand(1).getReg(); 5575 unsigned Offset = MI.getOperand(2).getImm(); 5576 5577 LLT DstTy = MRI.getType(Dst); 5578 LLT SrcTy = MRI.getType(Src); 5579 5580 if (DstTy.isScalar() && 5581 (SrcTy.isScalar() || 5582 (SrcTy.isVector() && DstTy == SrcTy.getElementType()))) { 5583 LLT SrcIntTy = SrcTy; 5584 if (!SrcTy.isScalar()) { 5585 SrcIntTy = LLT::scalar(SrcTy.getSizeInBits()); 5586 Src = MIRBuilder.buildBitcast(SrcIntTy, Src).getReg(0); 5587 } 5588 5589 if (Offset == 0) 5590 MIRBuilder.buildTrunc(Dst, Src); 5591 else { 5592 auto ShiftAmt = MIRBuilder.buildConstant(SrcIntTy, Offset); 5593 auto Shr = MIRBuilder.buildLShr(SrcIntTy, Src, ShiftAmt); 5594 MIRBuilder.buildTrunc(Dst, Shr); 5595 } 5596 5597 MI.eraseFromParent(); 5598 return Legalized; 5599 } 5600 5601 return UnableToLegalize; 5602 } 5603 5604 LegalizerHelper::LegalizeResult LegalizerHelper::lowerInsert(MachineInstr &MI) { 5605 Register Dst = MI.getOperand(0).getReg(); 5606 Register Src = MI.getOperand(1).getReg(); 5607 Register InsertSrc = MI.getOperand(2).getReg(); 5608 uint64_t Offset = MI.getOperand(3).getImm(); 5609 5610 LLT DstTy = MRI.getType(Src); 5611 LLT InsertTy = MRI.getType(InsertSrc); 5612 5613 if (InsertTy.isVector() || 5614 (DstTy.isVector() && DstTy.getElementType() != InsertTy)) 5615 return UnableToLegalize; 5616 5617 const DataLayout &DL = MIRBuilder.getDataLayout(); 5618 if ((DstTy.isPointer() && 5619 DL.isNonIntegralAddressSpace(DstTy.getAddressSpace())) || 5620 (InsertTy.isPointer() && 5621 DL.isNonIntegralAddressSpace(InsertTy.getAddressSpace()))) { 5622 LLVM_DEBUG(dbgs() << "Not casting non-integral address space integer\n"); 5623 return UnableToLegalize; 5624 } 5625 5626 LLT IntDstTy = DstTy; 5627 5628 if (!DstTy.isScalar()) { 5629 IntDstTy = LLT::scalar(DstTy.getSizeInBits()); 5630 Src = MIRBuilder.buildCast(IntDstTy, Src).getReg(0); 5631 } 5632 5633 if (!InsertTy.isScalar()) { 5634 const LLT IntInsertTy = LLT::scalar(InsertTy.getSizeInBits()); 5635 InsertSrc = MIRBuilder.buildPtrToInt(IntInsertTy, InsertSrc).getReg(0); 5636 } 5637 5638 Register ExtInsSrc = MIRBuilder.buildZExt(IntDstTy, InsertSrc).getReg(0); 5639 if (Offset != 0) { 5640 auto ShiftAmt = MIRBuilder.buildConstant(IntDstTy, Offset); 5641 ExtInsSrc = MIRBuilder.buildShl(IntDstTy, ExtInsSrc, ShiftAmt).getReg(0); 5642 } 5643 5644 APInt MaskVal = APInt::getBitsSetWithWrap( 5645 DstTy.getSizeInBits(), Offset + InsertTy.getSizeInBits(), Offset); 5646 5647 auto Mask = MIRBuilder.buildConstant(IntDstTy, MaskVal); 5648 auto MaskedSrc = MIRBuilder.buildAnd(IntDstTy, Src, Mask); 5649 auto Or = MIRBuilder.buildOr(IntDstTy, MaskedSrc, ExtInsSrc); 5650 5651 MIRBuilder.buildCast(Dst, Or); 5652 MI.eraseFromParent(); 5653 return Legalized; 5654 } 5655 5656 LegalizerHelper::LegalizeResult 5657 LegalizerHelper::lowerSADDO_SSUBO(MachineInstr &MI) { 5658 Register Dst0 = MI.getOperand(0).getReg(); 5659 Register Dst1 = MI.getOperand(1).getReg(); 5660 Register LHS = MI.getOperand(2).getReg(); 5661 Register RHS = MI.getOperand(3).getReg(); 5662 const bool IsAdd = MI.getOpcode() == TargetOpcode::G_SADDO; 5663 5664 LLT Ty = MRI.getType(Dst0); 5665 LLT BoolTy = MRI.getType(Dst1); 5666 5667 if (IsAdd) 5668 MIRBuilder.buildAdd(Dst0, LHS, RHS); 5669 else 5670 MIRBuilder.buildSub(Dst0, LHS, RHS); 5671 5672 // TODO: If SADDSAT/SSUBSAT is legal, compare results to detect overflow. 5673 5674 auto Zero = MIRBuilder.buildConstant(Ty, 0); 5675 5676 // For an addition, the result should be less than one of the operands (LHS) 5677 // if and only if the other operand (RHS) is negative, otherwise there will 5678 // be overflow. 5679 // For a subtraction, the result should be less than one of the operands 5680 // (LHS) if and only if the other operand (RHS) is (non-zero) positive, 5681 // otherwise there will be overflow. 5682 auto ResultLowerThanLHS = 5683 MIRBuilder.buildICmp(CmpInst::ICMP_SLT, BoolTy, Dst0, LHS); 5684 auto ConditionRHS = MIRBuilder.buildICmp( 5685 IsAdd ? CmpInst::ICMP_SLT : CmpInst::ICMP_SGT, BoolTy, RHS, Zero); 5686 5687 MIRBuilder.buildXor(Dst1, ConditionRHS, ResultLowerThanLHS); 5688 MI.eraseFromParent(); 5689 return Legalized; 5690 } 5691 5692 LegalizerHelper::LegalizeResult 5693 LegalizerHelper::lowerAddSubSatToMinMax(MachineInstr &MI) { 5694 Register Res = MI.getOperand(0).getReg(); 5695 Register LHS = MI.getOperand(1).getReg(); 5696 Register RHS = MI.getOperand(2).getReg(); 5697 LLT Ty = MRI.getType(Res); 5698 bool IsSigned; 5699 bool IsAdd; 5700 unsigned BaseOp; 5701 switch (MI.getOpcode()) { 5702 default: 5703 llvm_unreachable("unexpected addsat/subsat opcode"); 5704 case TargetOpcode::G_UADDSAT: 5705 IsSigned = false; 5706 IsAdd = true; 5707 BaseOp = TargetOpcode::G_ADD; 5708 break; 5709 case TargetOpcode::G_SADDSAT: 5710 IsSigned = true; 5711 IsAdd = true; 5712 BaseOp = TargetOpcode::G_ADD; 5713 break; 5714 case TargetOpcode::G_USUBSAT: 5715 IsSigned = false; 5716 IsAdd = false; 5717 BaseOp = TargetOpcode::G_SUB; 5718 break; 5719 case TargetOpcode::G_SSUBSAT: 5720 IsSigned = true; 5721 IsAdd = false; 5722 BaseOp = TargetOpcode::G_SUB; 5723 break; 5724 } 5725 5726 if (IsSigned) { 5727 // sadd.sat(a, b) -> 5728 // hi = 0x7fffffff - smax(a, 0) 5729 // lo = 0x80000000 - smin(a, 0) 5730 // a + smin(smax(lo, b), hi) 5731 // ssub.sat(a, b) -> 5732 // lo = smax(a, -1) - 0x7fffffff 5733 // hi = smin(a, -1) - 0x80000000 5734 // a - smin(smax(lo, b), hi) 5735 // TODO: AMDGPU can use a "median of 3" instruction here: 5736 // a +/- med3(lo, b, hi) 5737 uint64_t NumBits = Ty.getScalarSizeInBits(); 5738 auto MaxVal = 5739 MIRBuilder.buildConstant(Ty, APInt::getSignedMaxValue(NumBits)); 5740 auto MinVal = 5741 MIRBuilder.buildConstant(Ty, APInt::getSignedMinValue(NumBits)); 5742 MachineInstrBuilder Hi, Lo; 5743 if (IsAdd) { 5744 auto Zero = MIRBuilder.buildConstant(Ty, 0); 5745 Hi = MIRBuilder.buildSub(Ty, MaxVal, MIRBuilder.buildSMax(Ty, LHS, Zero)); 5746 Lo = MIRBuilder.buildSub(Ty, MinVal, MIRBuilder.buildSMin(Ty, LHS, Zero)); 5747 } else { 5748 auto NegOne = MIRBuilder.buildConstant(Ty, -1); 5749 Lo = MIRBuilder.buildSub(Ty, MIRBuilder.buildSMax(Ty, LHS, NegOne), 5750 MaxVal); 5751 Hi = MIRBuilder.buildSub(Ty, MIRBuilder.buildSMin(Ty, LHS, NegOne), 5752 MinVal); 5753 } 5754 auto RHSClamped = 5755 MIRBuilder.buildSMin(Ty, MIRBuilder.buildSMax(Ty, Lo, RHS), Hi); 5756 MIRBuilder.buildInstr(BaseOp, {Res}, {LHS, RHSClamped}); 5757 } else { 5758 // uadd.sat(a, b) -> a + umin(~a, b) 5759 // usub.sat(a, b) -> a - umin(a, b) 5760 Register Not = IsAdd ? MIRBuilder.buildNot(Ty, LHS).getReg(0) : LHS; 5761 auto Min = MIRBuilder.buildUMin(Ty, Not, RHS); 5762 MIRBuilder.buildInstr(BaseOp, {Res}, {LHS, Min}); 5763 } 5764 5765 MI.eraseFromParent(); 5766 return Legalized; 5767 } 5768 5769 LegalizerHelper::LegalizeResult 5770 LegalizerHelper::lowerAddSubSatToAddoSubo(MachineInstr &MI) { 5771 Register Res = MI.getOperand(0).getReg(); 5772 Register LHS = MI.getOperand(1).getReg(); 5773 Register RHS = MI.getOperand(2).getReg(); 5774 LLT Ty = MRI.getType(Res); 5775 LLT BoolTy = Ty.changeElementSize(1); 5776 bool IsSigned; 5777 bool IsAdd; 5778 unsigned OverflowOp; 5779 switch (MI.getOpcode()) { 5780 default: 5781 llvm_unreachable("unexpected addsat/subsat opcode"); 5782 case TargetOpcode::G_UADDSAT: 5783 IsSigned = false; 5784 IsAdd = true; 5785 OverflowOp = TargetOpcode::G_UADDO; 5786 break; 5787 case TargetOpcode::G_SADDSAT: 5788 IsSigned = true; 5789 IsAdd = true; 5790 OverflowOp = TargetOpcode::G_SADDO; 5791 break; 5792 case TargetOpcode::G_USUBSAT: 5793 IsSigned = false; 5794 IsAdd = false; 5795 OverflowOp = TargetOpcode::G_USUBO; 5796 break; 5797 case TargetOpcode::G_SSUBSAT: 5798 IsSigned = true; 5799 IsAdd = false; 5800 OverflowOp = TargetOpcode::G_SSUBO; 5801 break; 5802 } 5803 5804 auto OverflowRes = 5805 MIRBuilder.buildInstr(OverflowOp, {Ty, BoolTy}, {LHS, RHS}); 5806 Register Tmp = OverflowRes.getReg(0); 5807 Register Ov = OverflowRes.getReg(1); 5808 MachineInstrBuilder Clamp; 5809 if (IsSigned) { 5810 // sadd.sat(a, b) -> 5811 // {tmp, ov} = saddo(a, b) 5812 // ov ? (tmp >>s 31) + 0x80000000 : r 5813 // ssub.sat(a, b) -> 5814 // {tmp, ov} = ssubo(a, b) 5815 // ov ? (tmp >>s 31) + 0x80000000 : r 5816 uint64_t NumBits = Ty.getScalarSizeInBits(); 5817 auto ShiftAmount = MIRBuilder.buildConstant(Ty, NumBits - 1); 5818 auto Sign = MIRBuilder.buildAShr(Ty, Tmp, ShiftAmount); 5819 auto MinVal = 5820 MIRBuilder.buildConstant(Ty, APInt::getSignedMinValue(NumBits)); 5821 Clamp = MIRBuilder.buildAdd(Ty, Sign, MinVal); 5822 } else { 5823 // uadd.sat(a, b) -> 5824 // {tmp, ov} = uaddo(a, b) 5825 // ov ? 0xffffffff : tmp 5826 // usub.sat(a, b) -> 5827 // {tmp, ov} = usubo(a, b) 5828 // ov ? 0 : tmp 5829 Clamp = MIRBuilder.buildConstant(Ty, IsAdd ? -1 : 0); 5830 } 5831 MIRBuilder.buildSelect(Res, Ov, Clamp, Tmp); 5832 5833 MI.eraseFromParent(); 5834 return Legalized; 5835 } 5836 5837 LegalizerHelper::LegalizeResult 5838 LegalizerHelper::lowerBswap(MachineInstr &MI) { 5839 Register Dst = MI.getOperand(0).getReg(); 5840 Register Src = MI.getOperand(1).getReg(); 5841 const LLT Ty = MRI.getType(Src); 5842 unsigned SizeInBytes = (Ty.getScalarSizeInBits() + 7) / 8; 5843 unsigned BaseShiftAmt = (SizeInBytes - 1) * 8; 5844 5845 // Swap most and least significant byte, set remaining bytes in Res to zero. 5846 auto ShiftAmt = MIRBuilder.buildConstant(Ty, BaseShiftAmt); 5847 auto LSByteShiftedLeft = MIRBuilder.buildShl(Ty, Src, ShiftAmt); 5848 auto MSByteShiftedRight = MIRBuilder.buildLShr(Ty, Src, ShiftAmt); 5849 auto Res = MIRBuilder.buildOr(Ty, MSByteShiftedRight, LSByteShiftedLeft); 5850 5851 // Set i-th high/low byte in Res to i-th low/high byte from Src. 5852 for (unsigned i = 1; i < SizeInBytes / 2; ++i) { 5853 // AND with Mask leaves byte i unchanged and sets remaining bytes to 0. 5854 APInt APMask(SizeInBytes * 8, 0xFF << (i * 8)); 5855 auto Mask = MIRBuilder.buildConstant(Ty, APMask); 5856 auto ShiftAmt = MIRBuilder.buildConstant(Ty, BaseShiftAmt - 16 * i); 5857 // Low byte shifted left to place of high byte: (Src & Mask) << ShiftAmt. 5858 auto LoByte = MIRBuilder.buildAnd(Ty, Src, Mask); 5859 auto LoShiftedLeft = MIRBuilder.buildShl(Ty, LoByte, ShiftAmt); 5860 Res = MIRBuilder.buildOr(Ty, Res, LoShiftedLeft); 5861 // High byte shifted right to place of low byte: (Src >> ShiftAmt) & Mask. 5862 auto SrcShiftedRight = MIRBuilder.buildLShr(Ty, Src, ShiftAmt); 5863 auto HiShiftedRight = MIRBuilder.buildAnd(Ty, SrcShiftedRight, Mask); 5864 Res = MIRBuilder.buildOr(Ty, Res, HiShiftedRight); 5865 } 5866 Res.getInstr()->getOperand(0).setReg(Dst); 5867 5868 MI.eraseFromParent(); 5869 return Legalized; 5870 } 5871 5872 //{ (Src & Mask) >> N } | { (Src << N) & Mask } 5873 static MachineInstrBuilder SwapN(unsigned N, DstOp Dst, MachineIRBuilder &B, 5874 MachineInstrBuilder Src, APInt Mask) { 5875 const LLT Ty = Dst.getLLTTy(*B.getMRI()); 5876 MachineInstrBuilder C_N = B.buildConstant(Ty, N); 5877 MachineInstrBuilder MaskLoNTo0 = B.buildConstant(Ty, Mask); 5878 auto LHS = B.buildLShr(Ty, B.buildAnd(Ty, Src, MaskLoNTo0), C_N); 5879 auto RHS = B.buildAnd(Ty, B.buildShl(Ty, Src, C_N), MaskLoNTo0); 5880 return B.buildOr(Dst, LHS, RHS); 5881 } 5882 5883 LegalizerHelper::LegalizeResult 5884 LegalizerHelper::lowerBitreverse(MachineInstr &MI) { 5885 Register Dst = MI.getOperand(0).getReg(); 5886 Register Src = MI.getOperand(1).getReg(); 5887 const LLT Ty = MRI.getType(Src); 5888 unsigned Size = Ty.getSizeInBits(); 5889 5890 MachineInstrBuilder BSWAP = 5891 MIRBuilder.buildInstr(TargetOpcode::G_BSWAP, {Ty}, {Src}); 5892 5893 // swap high and low 4 bits in 8 bit blocks 7654|3210 -> 3210|7654 5894 // [(val & 0xF0F0F0F0) >> 4] | [(val & 0x0F0F0F0F) << 4] 5895 // -> [(val & 0xF0F0F0F0) >> 4] | [(val << 4) & 0xF0F0F0F0] 5896 MachineInstrBuilder Swap4 = 5897 SwapN(4, Ty, MIRBuilder, BSWAP, APInt::getSplat(Size, APInt(8, 0xF0))); 5898 5899 // swap high and low 2 bits in 4 bit blocks 32|10 76|54 -> 10|32 54|76 5900 // [(val & 0xCCCCCCCC) >> 2] & [(val & 0x33333333) << 2] 5901 // -> [(val & 0xCCCCCCCC) >> 2] & [(val << 2) & 0xCCCCCCCC] 5902 MachineInstrBuilder Swap2 = 5903 SwapN(2, Ty, MIRBuilder, Swap4, APInt::getSplat(Size, APInt(8, 0xCC))); 5904 5905 // swap high and low 1 bit in 2 bit blocks 1|0 3|2 5|4 7|6 -> 0|1 2|3 4|5 6|7 5906 // [(val & 0xAAAAAAAA) >> 1] & [(val & 0x55555555) << 1] 5907 // -> [(val & 0xAAAAAAAA) >> 1] & [(val << 1) & 0xAAAAAAAA] 5908 SwapN(1, Dst, MIRBuilder, Swap2, APInt::getSplat(Size, APInt(8, 0xAA))); 5909 5910 MI.eraseFromParent(); 5911 return Legalized; 5912 } 5913 5914 LegalizerHelper::LegalizeResult 5915 LegalizerHelper::lowerReadWriteRegister(MachineInstr &MI) { 5916 MachineFunction &MF = MIRBuilder.getMF(); 5917 const TargetSubtargetInfo &STI = MF.getSubtarget(); 5918 const TargetLowering *TLI = STI.getTargetLowering(); 5919 5920 bool IsRead = MI.getOpcode() == TargetOpcode::G_READ_REGISTER; 5921 int NameOpIdx = IsRead ? 1 : 0; 5922 int ValRegIndex = IsRead ? 0 : 1; 5923 5924 Register ValReg = MI.getOperand(ValRegIndex).getReg(); 5925 const LLT Ty = MRI.getType(ValReg); 5926 const MDString *RegStr = cast<MDString>( 5927 cast<MDNode>(MI.getOperand(NameOpIdx).getMetadata())->getOperand(0)); 5928 5929 Register PhysReg = TLI->getRegisterByName(RegStr->getString().data(), Ty, MF); 5930 if (!PhysReg.isValid()) 5931 return UnableToLegalize; 5932 5933 if (IsRead) 5934 MIRBuilder.buildCopy(ValReg, PhysReg); 5935 else 5936 MIRBuilder.buildCopy(PhysReg, ValReg); 5937 5938 MI.eraseFromParent(); 5939 return Legalized; 5940 } 5941