1 //===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file This file implements the LegalizerHelper class to legalize 10 /// individual instructions and the LegalizeMachineIR wrapper pass for the 11 /// primary legalization. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h" 16 #include "llvm/CodeGen/GlobalISel/CallLowering.h" 17 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" 18 #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" 19 #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" 20 #include "llvm/CodeGen/MachineRegisterInfo.h" 21 #include "llvm/CodeGen/TargetFrameLowering.h" 22 #include "llvm/CodeGen/TargetInstrInfo.h" 23 #include "llvm/CodeGen/TargetLowering.h" 24 #include "llvm/CodeGen/TargetSubtargetInfo.h" 25 #include "llvm/Support/Debug.h" 26 #include "llvm/Support/MathExtras.h" 27 #include "llvm/Support/raw_ostream.h" 28 29 #define DEBUG_TYPE "legalizer" 30 31 using namespace llvm; 32 using namespace LegalizeActions; 33 using namespace MIPatternMatch; 34 35 /// Try to break down \p OrigTy into \p NarrowTy sized pieces. 36 /// 37 /// Returns the number of \p NarrowTy elements needed to reconstruct \p OrigTy, 38 /// with any leftover piece as type \p LeftoverTy 39 /// 40 /// Returns -1 in the first element of the pair if the breakdown is not 41 /// satisfiable. 42 static std::pair<int, int> 43 getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) { 44 assert(!LeftoverTy.isValid() && "this is an out argument"); 45 46 unsigned Size = OrigTy.getSizeInBits(); 47 unsigned NarrowSize = NarrowTy.getSizeInBits(); 48 unsigned NumParts = Size / NarrowSize; 49 unsigned LeftoverSize = Size - NumParts * NarrowSize; 50 assert(Size > NarrowSize); 51 52 if (LeftoverSize == 0) 53 return {NumParts, 0}; 54 55 if (NarrowTy.isVector()) { 56 unsigned EltSize = OrigTy.getScalarSizeInBits(); 57 if (LeftoverSize % EltSize != 0) 58 return {-1, -1}; 59 LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize); 60 } else { 61 LeftoverTy = LLT::scalar(LeftoverSize); 62 } 63 64 int NumLeftover = LeftoverSize / LeftoverTy.getSizeInBits(); 65 return std::make_pair(NumParts, NumLeftover); 66 } 67 68 static Type *getFloatTypeForLLT(LLVMContext &Ctx, LLT Ty) { 69 70 if (!Ty.isScalar()) 71 return nullptr; 72 73 switch (Ty.getSizeInBits()) { 74 case 16: 75 return Type::getHalfTy(Ctx); 76 case 32: 77 return Type::getFloatTy(Ctx); 78 case 64: 79 return Type::getDoubleTy(Ctx); 80 case 80: 81 return Type::getX86_FP80Ty(Ctx); 82 case 128: 83 return Type::getFP128Ty(Ctx); 84 default: 85 return nullptr; 86 } 87 } 88 89 LegalizerHelper::LegalizerHelper(MachineFunction &MF, 90 GISelChangeObserver &Observer, 91 MachineIRBuilder &Builder) 92 : MIRBuilder(Builder), Observer(Observer), MRI(MF.getRegInfo()), 93 LI(*MF.getSubtarget().getLegalizerInfo()) { 94 MIRBuilder.setChangeObserver(Observer); 95 } 96 97 LegalizerHelper::LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI, 98 GISelChangeObserver &Observer, 99 MachineIRBuilder &B) 100 : MIRBuilder(B), Observer(Observer), MRI(MF.getRegInfo()), LI(LI) { 101 MIRBuilder.setChangeObserver(Observer); 102 } 103 LegalizerHelper::LegalizeResult 104 LegalizerHelper::legalizeInstrStep(MachineInstr &MI) { 105 LLVM_DEBUG(dbgs() << "Legalizing: " << MI); 106 107 MIRBuilder.setInstrAndDebugLoc(MI); 108 109 if (MI.getOpcode() == TargetOpcode::G_INTRINSIC || 110 MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS) 111 return LI.legalizeIntrinsic(*this, MI) ? Legalized : UnableToLegalize; 112 auto Step = LI.getAction(MI, MRI); 113 switch (Step.Action) { 114 case Legal: 115 LLVM_DEBUG(dbgs() << ".. Already legal\n"); 116 return AlreadyLegal; 117 case Libcall: 118 LLVM_DEBUG(dbgs() << ".. Convert to libcall\n"); 119 return libcall(MI); 120 case NarrowScalar: 121 LLVM_DEBUG(dbgs() << ".. Narrow scalar\n"); 122 return narrowScalar(MI, Step.TypeIdx, Step.NewType); 123 case WidenScalar: 124 LLVM_DEBUG(dbgs() << ".. Widen scalar\n"); 125 return widenScalar(MI, Step.TypeIdx, Step.NewType); 126 case Bitcast: 127 LLVM_DEBUG(dbgs() << ".. Bitcast type\n"); 128 return bitcast(MI, Step.TypeIdx, Step.NewType); 129 case Lower: 130 LLVM_DEBUG(dbgs() << ".. Lower\n"); 131 return lower(MI, Step.TypeIdx, Step.NewType); 132 case FewerElements: 133 LLVM_DEBUG(dbgs() << ".. Reduce number of elements\n"); 134 return fewerElementsVector(MI, Step.TypeIdx, Step.NewType); 135 case MoreElements: 136 LLVM_DEBUG(dbgs() << ".. Increase number of elements\n"); 137 return moreElementsVector(MI, Step.TypeIdx, Step.NewType); 138 case Custom: 139 LLVM_DEBUG(dbgs() << ".. Custom legalization\n"); 140 return LI.legalizeCustom(*this, MI) ? Legalized : UnableToLegalize; 141 default: 142 LLVM_DEBUG(dbgs() << ".. Unable to legalize\n"); 143 return UnableToLegalize; 144 } 145 } 146 147 void LegalizerHelper::extractParts(Register Reg, LLT Ty, int NumParts, 148 SmallVectorImpl<Register> &VRegs) { 149 for (int i = 0; i < NumParts; ++i) 150 VRegs.push_back(MRI.createGenericVirtualRegister(Ty)); 151 MIRBuilder.buildUnmerge(VRegs, Reg); 152 } 153 154 bool LegalizerHelper::extractParts(Register Reg, LLT RegTy, 155 LLT MainTy, LLT &LeftoverTy, 156 SmallVectorImpl<Register> &VRegs, 157 SmallVectorImpl<Register> &LeftoverRegs) { 158 assert(!LeftoverTy.isValid() && "this is an out argument"); 159 160 unsigned RegSize = RegTy.getSizeInBits(); 161 unsigned MainSize = MainTy.getSizeInBits(); 162 unsigned NumParts = RegSize / MainSize; 163 unsigned LeftoverSize = RegSize - NumParts * MainSize; 164 165 // Use an unmerge when possible. 166 if (LeftoverSize == 0) { 167 for (unsigned I = 0; I < NumParts; ++I) 168 VRegs.push_back(MRI.createGenericVirtualRegister(MainTy)); 169 MIRBuilder.buildUnmerge(VRegs, Reg); 170 return true; 171 } 172 173 if (MainTy.isVector()) { 174 unsigned EltSize = MainTy.getScalarSizeInBits(); 175 if (LeftoverSize % EltSize != 0) 176 return false; 177 LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize); 178 } else { 179 LeftoverTy = LLT::scalar(LeftoverSize); 180 } 181 182 // For irregular sizes, extract the individual parts. 183 for (unsigned I = 0; I != NumParts; ++I) { 184 Register NewReg = MRI.createGenericVirtualRegister(MainTy); 185 VRegs.push_back(NewReg); 186 MIRBuilder.buildExtract(NewReg, Reg, MainSize * I); 187 } 188 189 for (unsigned Offset = MainSize * NumParts; Offset < RegSize; 190 Offset += LeftoverSize) { 191 Register NewReg = MRI.createGenericVirtualRegister(LeftoverTy); 192 LeftoverRegs.push_back(NewReg); 193 MIRBuilder.buildExtract(NewReg, Reg, Offset); 194 } 195 196 return true; 197 } 198 199 void LegalizerHelper::insertParts(Register DstReg, 200 LLT ResultTy, LLT PartTy, 201 ArrayRef<Register> PartRegs, 202 LLT LeftoverTy, 203 ArrayRef<Register> LeftoverRegs) { 204 if (!LeftoverTy.isValid()) { 205 assert(LeftoverRegs.empty()); 206 207 if (!ResultTy.isVector()) { 208 MIRBuilder.buildMerge(DstReg, PartRegs); 209 return; 210 } 211 212 if (PartTy.isVector()) 213 MIRBuilder.buildConcatVectors(DstReg, PartRegs); 214 else 215 MIRBuilder.buildBuildVector(DstReg, PartRegs); 216 return; 217 } 218 219 unsigned PartSize = PartTy.getSizeInBits(); 220 unsigned LeftoverPartSize = LeftoverTy.getSizeInBits(); 221 222 Register CurResultReg = MRI.createGenericVirtualRegister(ResultTy); 223 MIRBuilder.buildUndef(CurResultReg); 224 225 unsigned Offset = 0; 226 for (Register PartReg : PartRegs) { 227 Register NewResultReg = MRI.createGenericVirtualRegister(ResultTy); 228 MIRBuilder.buildInsert(NewResultReg, CurResultReg, PartReg, Offset); 229 CurResultReg = NewResultReg; 230 Offset += PartSize; 231 } 232 233 for (unsigned I = 0, E = LeftoverRegs.size(); I != E; ++I) { 234 // Use the original output register for the final insert to avoid a copy. 235 Register NewResultReg = (I + 1 == E) ? 236 DstReg : MRI.createGenericVirtualRegister(ResultTy); 237 238 MIRBuilder.buildInsert(NewResultReg, CurResultReg, LeftoverRegs[I], Offset); 239 CurResultReg = NewResultReg; 240 Offset += LeftoverPartSize; 241 } 242 } 243 244 /// Return the result registers of G_UNMERGE_VALUES \p MI in \p Regs 245 static void getUnmergeResults(SmallVectorImpl<Register> &Regs, 246 const MachineInstr &MI) { 247 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES); 248 249 const int NumResults = MI.getNumOperands() - 1; 250 Regs.resize(NumResults); 251 for (int I = 0; I != NumResults; ++I) 252 Regs[I] = MI.getOperand(I).getReg(); 253 } 254 255 LLT LegalizerHelper::extractGCDType(SmallVectorImpl<Register> &Parts, LLT DstTy, 256 LLT NarrowTy, Register SrcReg) { 257 LLT SrcTy = MRI.getType(SrcReg); 258 259 LLT GCDTy = getGCDType(getGCDType(SrcTy, NarrowTy), DstTy); 260 if (SrcTy == GCDTy) { 261 // If the source already evenly divides the result type, we don't need to do 262 // anything. 263 Parts.push_back(SrcReg); 264 } else { 265 // Need to split into common type sized pieces. 266 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg); 267 getUnmergeResults(Parts, *Unmerge); 268 } 269 270 return GCDTy; 271 } 272 273 LLT LegalizerHelper::buildLCMMergePieces(LLT DstTy, LLT NarrowTy, LLT GCDTy, 274 SmallVectorImpl<Register> &VRegs, 275 unsigned PadStrategy) { 276 LLT LCMTy = getLCMType(DstTy, NarrowTy); 277 278 int NumParts = LCMTy.getSizeInBits() / NarrowTy.getSizeInBits(); 279 int NumSubParts = NarrowTy.getSizeInBits() / GCDTy.getSizeInBits(); 280 int NumOrigSrc = VRegs.size(); 281 282 Register PadReg; 283 284 // Get a value we can use to pad the source value if the sources won't evenly 285 // cover the result type. 286 if (NumOrigSrc < NumParts * NumSubParts) { 287 if (PadStrategy == TargetOpcode::G_ZEXT) 288 PadReg = MIRBuilder.buildConstant(GCDTy, 0).getReg(0); 289 else if (PadStrategy == TargetOpcode::G_ANYEXT) 290 PadReg = MIRBuilder.buildUndef(GCDTy).getReg(0); 291 else { 292 assert(PadStrategy == TargetOpcode::G_SEXT); 293 294 // Shift the sign bit of the low register through the high register. 295 auto ShiftAmt = 296 MIRBuilder.buildConstant(LLT::scalar(64), GCDTy.getSizeInBits() - 1); 297 PadReg = MIRBuilder.buildAShr(GCDTy, VRegs.back(), ShiftAmt).getReg(0); 298 } 299 } 300 301 // Registers for the final merge to be produced. 302 SmallVector<Register, 4> Remerge(NumParts); 303 304 // Registers needed for intermediate merges, which will be merged into a 305 // source for Remerge. 306 SmallVector<Register, 4> SubMerge(NumSubParts); 307 308 // Once we've fully read off the end of the original source bits, we can reuse 309 // the same high bits for remaining padding elements. 310 Register AllPadReg; 311 312 // Build merges to the LCM type to cover the original result type. 313 for (int I = 0; I != NumParts; ++I) { 314 bool AllMergePartsArePadding = true; 315 316 // Build the requested merges to the requested type. 317 for (int J = 0; J != NumSubParts; ++J) { 318 int Idx = I * NumSubParts + J; 319 if (Idx >= NumOrigSrc) { 320 SubMerge[J] = PadReg; 321 continue; 322 } 323 324 SubMerge[J] = VRegs[Idx]; 325 326 // There are meaningful bits here we can't reuse later. 327 AllMergePartsArePadding = false; 328 } 329 330 // If we've filled up a complete piece with padding bits, we can directly 331 // emit the natural sized constant if applicable, rather than a merge of 332 // smaller constants. 333 if (AllMergePartsArePadding && !AllPadReg) { 334 if (PadStrategy == TargetOpcode::G_ANYEXT) 335 AllPadReg = MIRBuilder.buildUndef(NarrowTy).getReg(0); 336 else if (PadStrategy == TargetOpcode::G_ZEXT) 337 AllPadReg = MIRBuilder.buildConstant(NarrowTy, 0).getReg(0); 338 339 // If this is a sign extension, we can't materialize a trivial constant 340 // with the right type and have to produce a merge. 341 } 342 343 if (AllPadReg) { 344 // Avoid creating additional instructions if we're just adding additional 345 // copies of padding bits. 346 Remerge[I] = AllPadReg; 347 continue; 348 } 349 350 if (NumSubParts == 1) 351 Remerge[I] = SubMerge[0]; 352 else 353 Remerge[I] = MIRBuilder.buildMerge(NarrowTy, SubMerge).getReg(0); 354 355 // In the sign extend padding case, re-use the first all-signbit merge. 356 if (AllMergePartsArePadding && !AllPadReg) 357 AllPadReg = Remerge[I]; 358 } 359 360 VRegs = std::move(Remerge); 361 return LCMTy; 362 } 363 364 void LegalizerHelper::buildWidenedRemergeToDst(Register DstReg, LLT LCMTy, 365 ArrayRef<Register> RemergeRegs) { 366 LLT DstTy = MRI.getType(DstReg); 367 368 // Create the merge to the widened source, and extract the relevant bits into 369 // the result. 370 371 if (DstTy == LCMTy) { 372 MIRBuilder.buildMerge(DstReg, RemergeRegs); 373 return; 374 } 375 376 auto Remerge = MIRBuilder.buildMerge(LCMTy, RemergeRegs); 377 if (DstTy.isScalar() && LCMTy.isScalar()) { 378 MIRBuilder.buildTrunc(DstReg, Remerge); 379 return; 380 } 381 382 if (LCMTy.isVector()) { 383 MIRBuilder.buildExtract(DstReg, Remerge, 0); 384 return; 385 } 386 387 llvm_unreachable("unhandled case"); 388 } 389 390 static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) { 391 #define RTLIBCASE_INT(LibcallPrefix) \ 392 do { \ 393 switch (Size) { \ 394 case 32: \ 395 return RTLIB::LibcallPrefix##32; \ 396 case 64: \ 397 return RTLIB::LibcallPrefix##64; \ 398 case 128: \ 399 return RTLIB::LibcallPrefix##128; \ 400 default: \ 401 llvm_unreachable("unexpected size"); \ 402 } \ 403 } while (0) 404 405 #define RTLIBCASE(LibcallPrefix) \ 406 do { \ 407 switch (Size) { \ 408 case 32: \ 409 return RTLIB::LibcallPrefix##32; \ 410 case 64: \ 411 return RTLIB::LibcallPrefix##64; \ 412 case 80: \ 413 return RTLIB::LibcallPrefix##80; \ 414 case 128: \ 415 return RTLIB::LibcallPrefix##128; \ 416 default: \ 417 llvm_unreachable("unexpected size"); \ 418 } \ 419 } while (0) 420 421 switch (Opcode) { 422 case TargetOpcode::G_SDIV: 423 RTLIBCASE_INT(SDIV_I); 424 case TargetOpcode::G_UDIV: 425 RTLIBCASE_INT(UDIV_I); 426 case TargetOpcode::G_SREM: 427 RTLIBCASE_INT(SREM_I); 428 case TargetOpcode::G_UREM: 429 RTLIBCASE_INT(UREM_I); 430 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 431 RTLIBCASE_INT(CTLZ_I); 432 case TargetOpcode::G_FADD: 433 RTLIBCASE(ADD_F); 434 case TargetOpcode::G_FSUB: 435 RTLIBCASE(SUB_F); 436 case TargetOpcode::G_FMUL: 437 RTLIBCASE(MUL_F); 438 case TargetOpcode::G_FDIV: 439 RTLIBCASE(DIV_F); 440 case TargetOpcode::G_FEXP: 441 RTLIBCASE(EXP_F); 442 case TargetOpcode::G_FEXP2: 443 RTLIBCASE(EXP2_F); 444 case TargetOpcode::G_FREM: 445 RTLIBCASE(REM_F); 446 case TargetOpcode::G_FPOW: 447 RTLIBCASE(POW_F); 448 case TargetOpcode::G_FMA: 449 RTLIBCASE(FMA_F); 450 case TargetOpcode::G_FSIN: 451 RTLIBCASE(SIN_F); 452 case TargetOpcode::G_FCOS: 453 RTLIBCASE(COS_F); 454 case TargetOpcode::G_FLOG10: 455 RTLIBCASE(LOG10_F); 456 case TargetOpcode::G_FLOG: 457 RTLIBCASE(LOG_F); 458 case TargetOpcode::G_FLOG2: 459 RTLIBCASE(LOG2_F); 460 case TargetOpcode::G_FCEIL: 461 RTLIBCASE(CEIL_F); 462 case TargetOpcode::G_FFLOOR: 463 RTLIBCASE(FLOOR_F); 464 case TargetOpcode::G_FMINNUM: 465 RTLIBCASE(FMIN_F); 466 case TargetOpcode::G_FMAXNUM: 467 RTLIBCASE(FMAX_F); 468 case TargetOpcode::G_FSQRT: 469 RTLIBCASE(SQRT_F); 470 case TargetOpcode::G_FRINT: 471 RTLIBCASE(RINT_F); 472 case TargetOpcode::G_FNEARBYINT: 473 RTLIBCASE(NEARBYINT_F); 474 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: 475 RTLIBCASE(ROUNDEVEN_F); 476 } 477 llvm_unreachable("Unknown libcall function"); 478 } 479 480 /// True if an instruction is in tail position in its caller. Intended for 481 /// legalizing libcalls as tail calls when possible. 482 static bool isLibCallInTailPosition(const TargetInstrInfo &TII, 483 MachineInstr &MI) { 484 MachineBasicBlock &MBB = *MI.getParent(); 485 const Function &F = MBB.getParent()->getFunction(); 486 487 // Conservatively require the attributes of the call to match those of 488 // the return. Ignore NoAlias and NonNull because they don't affect the 489 // call sequence. 490 AttributeList CallerAttrs = F.getAttributes(); 491 if (AttrBuilder(CallerAttrs, AttributeList::ReturnIndex) 492 .removeAttribute(Attribute::NoAlias) 493 .removeAttribute(Attribute::NonNull) 494 .hasAttributes()) 495 return false; 496 497 // It's not safe to eliminate the sign / zero extension of the return value. 498 if (CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::ZExt) || 499 CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::SExt)) 500 return false; 501 502 // Only tail call if the following instruction is a standard return. 503 auto Next = next_nodbg(MI.getIterator(), MBB.instr_end()); 504 if (Next == MBB.instr_end() || TII.isTailCall(*Next) || !Next->isReturn()) 505 return false; 506 507 return true; 508 } 509 510 LegalizerHelper::LegalizeResult 511 llvm::createLibcall(MachineIRBuilder &MIRBuilder, const char *Name, 512 const CallLowering::ArgInfo &Result, 513 ArrayRef<CallLowering::ArgInfo> Args, 514 const CallingConv::ID CC) { 515 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering(); 516 517 CallLowering::CallLoweringInfo Info; 518 Info.CallConv = CC; 519 Info.Callee = MachineOperand::CreateES(Name); 520 Info.OrigRet = Result; 521 std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs)); 522 if (!CLI.lowerCall(MIRBuilder, Info)) 523 return LegalizerHelper::UnableToLegalize; 524 525 return LegalizerHelper::Legalized; 526 } 527 528 LegalizerHelper::LegalizeResult 529 llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall, 530 const CallLowering::ArgInfo &Result, 531 ArrayRef<CallLowering::ArgInfo> Args) { 532 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering(); 533 const char *Name = TLI.getLibcallName(Libcall); 534 const CallingConv::ID CC = TLI.getLibcallCallingConv(Libcall); 535 return createLibcall(MIRBuilder, Name, Result, Args, CC); 536 } 537 538 // Useful for libcalls where all operands have the same type. 539 static LegalizerHelper::LegalizeResult 540 simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size, 541 Type *OpType) { 542 auto Libcall = getRTLibDesc(MI.getOpcode(), Size); 543 544 SmallVector<CallLowering::ArgInfo, 3> Args; 545 for (unsigned i = 1; i < MI.getNumOperands(); i++) 546 Args.push_back({MI.getOperand(i).getReg(), OpType}); 547 return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), OpType}, 548 Args); 549 } 550 551 LegalizerHelper::LegalizeResult 552 llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, 553 MachineInstr &MI) { 554 assert(MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS); 555 auto &Ctx = MIRBuilder.getMF().getFunction().getContext(); 556 557 SmallVector<CallLowering::ArgInfo, 3> Args; 558 // Add all the args, except for the last which is an imm denoting 'tail'. 559 for (unsigned i = 1; i < MI.getNumOperands() - 1; i++) { 560 Register Reg = MI.getOperand(i).getReg(); 561 562 // Need derive an IR type for call lowering. 563 LLT OpLLT = MRI.getType(Reg); 564 Type *OpTy = nullptr; 565 if (OpLLT.isPointer()) 566 OpTy = Type::getInt8PtrTy(Ctx, OpLLT.getAddressSpace()); 567 else 568 OpTy = IntegerType::get(Ctx, OpLLT.getSizeInBits()); 569 Args.push_back({Reg, OpTy}); 570 } 571 572 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering(); 573 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering(); 574 Intrinsic::ID ID = MI.getOperand(0).getIntrinsicID(); 575 RTLIB::Libcall RTLibcall; 576 switch (ID) { 577 case Intrinsic::memcpy: 578 RTLibcall = RTLIB::MEMCPY; 579 break; 580 case Intrinsic::memset: 581 RTLibcall = RTLIB::MEMSET; 582 break; 583 case Intrinsic::memmove: 584 RTLibcall = RTLIB::MEMMOVE; 585 break; 586 default: 587 return LegalizerHelper::UnableToLegalize; 588 } 589 const char *Name = TLI.getLibcallName(RTLibcall); 590 591 MIRBuilder.setInstrAndDebugLoc(MI); 592 593 CallLowering::CallLoweringInfo Info; 594 Info.CallConv = TLI.getLibcallCallingConv(RTLibcall); 595 Info.Callee = MachineOperand::CreateES(Name); 596 Info.OrigRet = CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx)); 597 Info.IsTailCall = MI.getOperand(MI.getNumOperands() - 1).getImm() == 1 && 598 isLibCallInTailPosition(MIRBuilder.getTII(), MI); 599 600 std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs)); 601 if (!CLI.lowerCall(MIRBuilder, Info)) 602 return LegalizerHelper::UnableToLegalize; 603 604 if (Info.LoweredTailCall) { 605 assert(Info.IsTailCall && "Lowered tail call when it wasn't a tail call?"); 606 // We must have a return following the call (or debug insts) to get past 607 // isLibCallInTailPosition. 608 do { 609 MachineInstr *Next = MI.getNextNode(); 610 assert(Next && (Next->isReturn() || Next->isDebugInstr()) && 611 "Expected instr following MI to be return or debug inst?"); 612 // We lowered a tail call, so the call is now the return from the block. 613 // Delete the old return. 614 Next->eraseFromParent(); 615 } while (MI.getNextNode()); 616 } 617 618 return LegalizerHelper::Legalized; 619 } 620 621 static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType, 622 Type *FromType) { 623 auto ToMVT = MVT::getVT(ToType); 624 auto FromMVT = MVT::getVT(FromType); 625 626 switch (Opcode) { 627 case TargetOpcode::G_FPEXT: 628 return RTLIB::getFPEXT(FromMVT, ToMVT); 629 case TargetOpcode::G_FPTRUNC: 630 return RTLIB::getFPROUND(FromMVT, ToMVT); 631 case TargetOpcode::G_FPTOSI: 632 return RTLIB::getFPTOSINT(FromMVT, ToMVT); 633 case TargetOpcode::G_FPTOUI: 634 return RTLIB::getFPTOUINT(FromMVT, ToMVT); 635 case TargetOpcode::G_SITOFP: 636 return RTLIB::getSINTTOFP(FromMVT, ToMVT); 637 case TargetOpcode::G_UITOFP: 638 return RTLIB::getUINTTOFP(FromMVT, ToMVT); 639 } 640 llvm_unreachable("Unsupported libcall function"); 641 } 642 643 static LegalizerHelper::LegalizeResult 644 conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType, 645 Type *FromType) { 646 RTLIB::Libcall Libcall = getConvRTLibDesc(MI.getOpcode(), ToType, FromType); 647 return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), ToType}, 648 {{MI.getOperand(1).getReg(), FromType}}); 649 } 650 651 LegalizerHelper::LegalizeResult 652 LegalizerHelper::libcall(MachineInstr &MI) { 653 LLT LLTy = MRI.getType(MI.getOperand(0).getReg()); 654 unsigned Size = LLTy.getSizeInBits(); 655 auto &Ctx = MIRBuilder.getMF().getFunction().getContext(); 656 657 switch (MI.getOpcode()) { 658 default: 659 return UnableToLegalize; 660 case TargetOpcode::G_SDIV: 661 case TargetOpcode::G_UDIV: 662 case TargetOpcode::G_SREM: 663 case TargetOpcode::G_UREM: 664 case TargetOpcode::G_CTLZ_ZERO_UNDEF: { 665 Type *HLTy = IntegerType::get(Ctx, Size); 666 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy); 667 if (Status != Legalized) 668 return Status; 669 break; 670 } 671 case TargetOpcode::G_FADD: 672 case TargetOpcode::G_FSUB: 673 case TargetOpcode::G_FMUL: 674 case TargetOpcode::G_FDIV: 675 case TargetOpcode::G_FMA: 676 case TargetOpcode::G_FPOW: 677 case TargetOpcode::G_FREM: 678 case TargetOpcode::G_FCOS: 679 case TargetOpcode::G_FSIN: 680 case TargetOpcode::G_FLOG10: 681 case TargetOpcode::G_FLOG: 682 case TargetOpcode::G_FLOG2: 683 case TargetOpcode::G_FEXP: 684 case TargetOpcode::G_FEXP2: 685 case TargetOpcode::G_FCEIL: 686 case TargetOpcode::G_FFLOOR: 687 case TargetOpcode::G_FMINNUM: 688 case TargetOpcode::G_FMAXNUM: 689 case TargetOpcode::G_FSQRT: 690 case TargetOpcode::G_FRINT: 691 case TargetOpcode::G_FNEARBYINT: 692 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: { 693 Type *HLTy = getFloatTypeForLLT(Ctx, LLTy); 694 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) { 695 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n"); 696 return UnableToLegalize; 697 } 698 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy); 699 if (Status != Legalized) 700 return Status; 701 break; 702 } 703 case TargetOpcode::G_FPEXT: 704 case TargetOpcode::G_FPTRUNC: { 705 Type *FromTy = getFloatTypeForLLT(Ctx, MRI.getType(MI.getOperand(1).getReg())); 706 Type *ToTy = getFloatTypeForLLT(Ctx, MRI.getType(MI.getOperand(0).getReg())); 707 if (!FromTy || !ToTy) 708 return UnableToLegalize; 709 LegalizeResult Status = conversionLibcall(MI, MIRBuilder, ToTy, FromTy ); 710 if (Status != Legalized) 711 return Status; 712 break; 713 } 714 case TargetOpcode::G_FPTOSI: 715 case TargetOpcode::G_FPTOUI: { 716 // FIXME: Support other types 717 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits(); 718 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 719 if ((ToSize != 32 && ToSize != 64) || (FromSize != 32 && FromSize != 64)) 720 return UnableToLegalize; 721 LegalizeResult Status = conversionLibcall( 722 MI, MIRBuilder, 723 ToSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx), 724 FromSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx)); 725 if (Status != Legalized) 726 return Status; 727 break; 728 } 729 case TargetOpcode::G_SITOFP: 730 case TargetOpcode::G_UITOFP: { 731 // FIXME: Support other types 732 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits(); 733 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 734 if ((FromSize != 32 && FromSize != 64) || (ToSize != 32 && ToSize != 64)) 735 return UnableToLegalize; 736 LegalizeResult Status = conversionLibcall( 737 MI, MIRBuilder, 738 ToSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx), 739 FromSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx)); 740 if (Status != Legalized) 741 return Status; 742 break; 743 } 744 } 745 746 MI.eraseFromParent(); 747 return Legalized; 748 } 749 750 LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI, 751 unsigned TypeIdx, 752 LLT NarrowTy) { 753 uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 754 uint64_t NarrowSize = NarrowTy.getSizeInBits(); 755 756 switch (MI.getOpcode()) { 757 default: 758 return UnableToLegalize; 759 case TargetOpcode::G_IMPLICIT_DEF: { 760 Register DstReg = MI.getOperand(0).getReg(); 761 LLT DstTy = MRI.getType(DstReg); 762 763 // If SizeOp0 is not an exact multiple of NarrowSize, emit 764 // G_ANYEXT(G_IMPLICIT_DEF). Cast result to vector if needed. 765 // FIXME: Although this would also be legal for the general case, it causes 766 // a lot of regressions in the emitted code (superfluous COPYs, artifact 767 // combines not being hit). This seems to be a problem related to the 768 // artifact combiner. 769 if (SizeOp0 % NarrowSize != 0) { 770 LLT ImplicitTy = NarrowTy; 771 if (DstTy.isVector()) 772 ImplicitTy = LLT::vector(DstTy.getNumElements(), ImplicitTy); 773 774 Register ImplicitReg = MIRBuilder.buildUndef(ImplicitTy).getReg(0); 775 MIRBuilder.buildAnyExt(DstReg, ImplicitReg); 776 777 MI.eraseFromParent(); 778 return Legalized; 779 } 780 781 int NumParts = SizeOp0 / NarrowSize; 782 783 SmallVector<Register, 2> DstRegs; 784 for (int i = 0; i < NumParts; ++i) 785 DstRegs.push_back(MIRBuilder.buildUndef(NarrowTy).getReg(0)); 786 787 if (DstTy.isVector()) 788 MIRBuilder.buildBuildVector(DstReg, DstRegs); 789 else 790 MIRBuilder.buildMerge(DstReg, DstRegs); 791 MI.eraseFromParent(); 792 return Legalized; 793 } 794 case TargetOpcode::G_CONSTANT: { 795 LLT Ty = MRI.getType(MI.getOperand(0).getReg()); 796 const APInt &Val = MI.getOperand(1).getCImm()->getValue(); 797 unsigned TotalSize = Ty.getSizeInBits(); 798 unsigned NarrowSize = NarrowTy.getSizeInBits(); 799 int NumParts = TotalSize / NarrowSize; 800 801 SmallVector<Register, 4> PartRegs; 802 for (int I = 0; I != NumParts; ++I) { 803 unsigned Offset = I * NarrowSize; 804 auto K = MIRBuilder.buildConstant(NarrowTy, 805 Val.lshr(Offset).trunc(NarrowSize)); 806 PartRegs.push_back(K.getReg(0)); 807 } 808 809 LLT LeftoverTy; 810 unsigned LeftoverBits = TotalSize - NumParts * NarrowSize; 811 SmallVector<Register, 1> LeftoverRegs; 812 if (LeftoverBits != 0) { 813 LeftoverTy = LLT::scalar(LeftoverBits); 814 auto K = MIRBuilder.buildConstant( 815 LeftoverTy, 816 Val.lshr(NumParts * NarrowSize).trunc(LeftoverBits)); 817 LeftoverRegs.push_back(K.getReg(0)); 818 } 819 820 insertParts(MI.getOperand(0).getReg(), 821 Ty, NarrowTy, PartRegs, LeftoverTy, LeftoverRegs); 822 823 MI.eraseFromParent(); 824 return Legalized; 825 } 826 case TargetOpcode::G_SEXT: 827 case TargetOpcode::G_ZEXT: 828 case TargetOpcode::G_ANYEXT: 829 return narrowScalarExt(MI, TypeIdx, NarrowTy); 830 case TargetOpcode::G_TRUNC: { 831 if (TypeIdx != 1) 832 return UnableToLegalize; 833 834 uint64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits(); 835 if (NarrowTy.getSizeInBits() * 2 != SizeOp1) { 836 LLVM_DEBUG(dbgs() << "Can't narrow trunc to type " << NarrowTy << "\n"); 837 return UnableToLegalize; 838 } 839 840 auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1)); 841 MIRBuilder.buildCopy(MI.getOperand(0), Unmerge.getReg(0)); 842 MI.eraseFromParent(); 843 return Legalized; 844 } 845 846 case TargetOpcode::G_FREEZE: 847 return reduceOperationWidth(MI, TypeIdx, NarrowTy); 848 849 case TargetOpcode::G_ADD: { 850 // FIXME: add support for when SizeOp0 isn't an exact multiple of 851 // NarrowSize. 852 if (SizeOp0 % NarrowSize != 0) 853 return UnableToLegalize; 854 // Expand in terms of carry-setting/consuming G_ADDE instructions. 855 int NumParts = SizeOp0 / NarrowTy.getSizeInBits(); 856 857 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs; 858 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs); 859 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs); 860 861 Register CarryIn; 862 for (int i = 0; i < NumParts; ++i) { 863 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy); 864 Register CarryOut = MRI.createGenericVirtualRegister(LLT::scalar(1)); 865 866 if (i == 0) 867 MIRBuilder.buildUAddo(DstReg, CarryOut, Src1Regs[i], Src2Regs[i]); 868 else { 869 MIRBuilder.buildUAdde(DstReg, CarryOut, Src1Regs[i], 870 Src2Regs[i], CarryIn); 871 } 872 873 DstRegs.push_back(DstReg); 874 CarryIn = CarryOut; 875 } 876 Register DstReg = MI.getOperand(0).getReg(); 877 if(MRI.getType(DstReg).isVector()) 878 MIRBuilder.buildBuildVector(DstReg, DstRegs); 879 else 880 MIRBuilder.buildMerge(DstReg, DstRegs); 881 MI.eraseFromParent(); 882 return Legalized; 883 } 884 case TargetOpcode::G_SUB: { 885 // FIXME: add support for when SizeOp0 isn't an exact multiple of 886 // NarrowSize. 887 if (SizeOp0 % NarrowSize != 0) 888 return UnableToLegalize; 889 890 int NumParts = SizeOp0 / NarrowTy.getSizeInBits(); 891 892 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs; 893 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs); 894 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs); 895 896 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy); 897 Register BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1)); 898 MIRBuilder.buildInstr(TargetOpcode::G_USUBO, {DstReg, BorrowOut}, 899 {Src1Regs[0], Src2Regs[0]}); 900 DstRegs.push_back(DstReg); 901 Register BorrowIn = BorrowOut; 902 for (int i = 1; i < NumParts; ++i) { 903 DstReg = MRI.createGenericVirtualRegister(NarrowTy); 904 BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1)); 905 906 MIRBuilder.buildInstr(TargetOpcode::G_USUBE, {DstReg, BorrowOut}, 907 {Src1Regs[i], Src2Regs[i], BorrowIn}); 908 909 DstRegs.push_back(DstReg); 910 BorrowIn = BorrowOut; 911 } 912 MIRBuilder.buildMerge(MI.getOperand(0), DstRegs); 913 MI.eraseFromParent(); 914 return Legalized; 915 } 916 case TargetOpcode::G_MUL: 917 case TargetOpcode::G_UMULH: 918 return narrowScalarMul(MI, NarrowTy); 919 case TargetOpcode::G_EXTRACT: 920 return narrowScalarExtract(MI, TypeIdx, NarrowTy); 921 case TargetOpcode::G_INSERT: 922 return narrowScalarInsert(MI, TypeIdx, NarrowTy); 923 case TargetOpcode::G_LOAD: { 924 auto &MMO = **MI.memoperands_begin(); 925 Register DstReg = MI.getOperand(0).getReg(); 926 LLT DstTy = MRI.getType(DstReg); 927 if (DstTy.isVector()) 928 return UnableToLegalize; 929 930 if (8 * MMO.getSize() != DstTy.getSizeInBits()) { 931 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy); 932 MIRBuilder.buildLoad(TmpReg, MI.getOperand(1), MMO); 933 MIRBuilder.buildAnyExt(DstReg, TmpReg); 934 MI.eraseFromParent(); 935 return Legalized; 936 } 937 938 return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy); 939 } 940 case TargetOpcode::G_ZEXTLOAD: 941 case TargetOpcode::G_SEXTLOAD: { 942 bool ZExt = MI.getOpcode() == TargetOpcode::G_ZEXTLOAD; 943 Register DstReg = MI.getOperand(0).getReg(); 944 Register PtrReg = MI.getOperand(1).getReg(); 945 946 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy); 947 auto &MMO = **MI.memoperands_begin(); 948 if (MMO.getSizeInBits() == NarrowSize) { 949 MIRBuilder.buildLoad(TmpReg, PtrReg, MMO); 950 } else { 951 MIRBuilder.buildLoadInstr(MI.getOpcode(), TmpReg, PtrReg, MMO); 952 } 953 954 if (ZExt) 955 MIRBuilder.buildZExt(DstReg, TmpReg); 956 else 957 MIRBuilder.buildSExt(DstReg, TmpReg); 958 959 MI.eraseFromParent(); 960 return Legalized; 961 } 962 case TargetOpcode::G_STORE: { 963 const auto &MMO = **MI.memoperands_begin(); 964 965 Register SrcReg = MI.getOperand(0).getReg(); 966 LLT SrcTy = MRI.getType(SrcReg); 967 if (SrcTy.isVector()) 968 return UnableToLegalize; 969 970 int NumParts = SizeOp0 / NarrowSize; 971 unsigned HandledSize = NumParts * NarrowTy.getSizeInBits(); 972 unsigned LeftoverBits = SrcTy.getSizeInBits() - HandledSize; 973 if (SrcTy.isVector() && LeftoverBits != 0) 974 return UnableToLegalize; 975 976 if (8 * MMO.getSize() != SrcTy.getSizeInBits()) { 977 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy); 978 auto &MMO = **MI.memoperands_begin(); 979 MIRBuilder.buildTrunc(TmpReg, SrcReg); 980 MIRBuilder.buildStore(TmpReg, MI.getOperand(1), MMO); 981 MI.eraseFromParent(); 982 return Legalized; 983 } 984 985 return reduceLoadStoreWidth(MI, 0, NarrowTy); 986 } 987 case TargetOpcode::G_SELECT: 988 return narrowScalarSelect(MI, TypeIdx, NarrowTy); 989 case TargetOpcode::G_AND: 990 case TargetOpcode::G_OR: 991 case TargetOpcode::G_XOR: { 992 // Legalize bitwise operation: 993 // A = BinOp<Ty> B, C 994 // into: 995 // B1, ..., BN = G_UNMERGE_VALUES B 996 // C1, ..., CN = G_UNMERGE_VALUES C 997 // A1 = BinOp<Ty/N> B1, C2 998 // ... 999 // AN = BinOp<Ty/N> BN, CN 1000 // A = G_MERGE_VALUES A1, ..., AN 1001 return narrowScalarBasic(MI, TypeIdx, NarrowTy); 1002 } 1003 case TargetOpcode::G_SHL: 1004 case TargetOpcode::G_LSHR: 1005 case TargetOpcode::G_ASHR: 1006 return narrowScalarShift(MI, TypeIdx, NarrowTy); 1007 case TargetOpcode::G_CTLZ: 1008 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 1009 case TargetOpcode::G_CTTZ: 1010 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 1011 case TargetOpcode::G_CTPOP: 1012 if (TypeIdx == 1) 1013 switch (MI.getOpcode()) { 1014 case TargetOpcode::G_CTLZ: 1015 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 1016 return narrowScalarCTLZ(MI, TypeIdx, NarrowTy); 1017 case TargetOpcode::G_CTTZ: 1018 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 1019 return narrowScalarCTTZ(MI, TypeIdx, NarrowTy); 1020 case TargetOpcode::G_CTPOP: 1021 return narrowScalarCTPOP(MI, TypeIdx, NarrowTy); 1022 default: 1023 return UnableToLegalize; 1024 } 1025 1026 Observer.changingInstr(MI); 1027 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT); 1028 Observer.changedInstr(MI); 1029 return Legalized; 1030 case TargetOpcode::G_INTTOPTR: 1031 if (TypeIdx != 1) 1032 return UnableToLegalize; 1033 1034 Observer.changingInstr(MI); 1035 narrowScalarSrc(MI, NarrowTy, 1); 1036 Observer.changedInstr(MI); 1037 return Legalized; 1038 case TargetOpcode::G_PTRTOINT: 1039 if (TypeIdx != 0) 1040 return UnableToLegalize; 1041 1042 Observer.changingInstr(MI); 1043 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT); 1044 Observer.changedInstr(MI); 1045 return Legalized; 1046 case TargetOpcode::G_PHI: { 1047 unsigned NumParts = SizeOp0 / NarrowSize; 1048 SmallVector<Register, 2> DstRegs(NumParts); 1049 SmallVector<SmallVector<Register, 2>, 2> SrcRegs(MI.getNumOperands() / 2); 1050 Observer.changingInstr(MI); 1051 for (unsigned i = 1; i < MI.getNumOperands(); i += 2) { 1052 MachineBasicBlock &OpMBB = *MI.getOperand(i + 1).getMBB(); 1053 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator()); 1054 extractParts(MI.getOperand(i).getReg(), NarrowTy, NumParts, 1055 SrcRegs[i / 2]); 1056 } 1057 MachineBasicBlock &MBB = *MI.getParent(); 1058 MIRBuilder.setInsertPt(MBB, MI); 1059 for (unsigned i = 0; i < NumParts; ++i) { 1060 DstRegs[i] = MRI.createGenericVirtualRegister(NarrowTy); 1061 MachineInstrBuilder MIB = 1062 MIRBuilder.buildInstr(TargetOpcode::G_PHI).addDef(DstRegs[i]); 1063 for (unsigned j = 1; j < MI.getNumOperands(); j += 2) 1064 MIB.addUse(SrcRegs[j / 2][i]).add(MI.getOperand(j + 1)); 1065 } 1066 MIRBuilder.setInsertPt(MBB, MBB.getFirstNonPHI()); 1067 MIRBuilder.buildMerge(MI.getOperand(0), DstRegs); 1068 Observer.changedInstr(MI); 1069 MI.eraseFromParent(); 1070 return Legalized; 1071 } 1072 case TargetOpcode::G_EXTRACT_VECTOR_ELT: 1073 case TargetOpcode::G_INSERT_VECTOR_ELT: { 1074 if (TypeIdx != 2) 1075 return UnableToLegalize; 1076 1077 int OpIdx = MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3; 1078 Observer.changingInstr(MI); 1079 narrowScalarSrc(MI, NarrowTy, OpIdx); 1080 Observer.changedInstr(MI); 1081 return Legalized; 1082 } 1083 case TargetOpcode::G_ICMP: { 1084 uint64_t SrcSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits(); 1085 if (NarrowSize * 2 != SrcSize) 1086 return UnableToLegalize; 1087 1088 Observer.changingInstr(MI); 1089 Register LHSL = MRI.createGenericVirtualRegister(NarrowTy); 1090 Register LHSH = MRI.createGenericVirtualRegister(NarrowTy); 1091 MIRBuilder.buildUnmerge({LHSL, LHSH}, MI.getOperand(2)); 1092 1093 Register RHSL = MRI.createGenericVirtualRegister(NarrowTy); 1094 Register RHSH = MRI.createGenericVirtualRegister(NarrowTy); 1095 MIRBuilder.buildUnmerge({RHSL, RHSH}, MI.getOperand(3)); 1096 1097 CmpInst::Predicate Pred = 1098 static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate()); 1099 LLT ResTy = MRI.getType(MI.getOperand(0).getReg()); 1100 1101 if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE) { 1102 MachineInstrBuilder XorL = MIRBuilder.buildXor(NarrowTy, LHSL, RHSL); 1103 MachineInstrBuilder XorH = MIRBuilder.buildXor(NarrowTy, LHSH, RHSH); 1104 MachineInstrBuilder Or = MIRBuilder.buildOr(NarrowTy, XorL, XorH); 1105 MachineInstrBuilder Zero = MIRBuilder.buildConstant(NarrowTy, 0); 1106 MIRBuilder.buildICmp(Pred, MI.getOperand(0), Or, Zero); 1107 } else { 1108 MachineInstrBuilder CmpH = MIRBuilder.buildICmp(Pred, ResTy, LHSH, RHSH); 1109 MachineInstrBuilder CmpHEQ = 1110 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, ResTy, LHSH, RHSH); 1111 MachineInstrBuilder CmpLU = MIRBuilder.buildICmp( 1112 ICmpInst::getUnsignedPredicate(Pred), ResTy, LHSL, RHSL); 1113 MIRBuilder.buildSelect(MI.getOperand(0), CmpHEQ, CmpLU, CmpH); 1114 } 1115 Observer.changedInstr(MI); 1116 MI.eraseFromParent(); 1117 return Legalized; 1118 } 1119 case TargetOpcode::G_SEXT_INREG: { 1120 if (TypeIdx != 0) 1121 return UnableToLegalize; 1122 1123 int64_t SizeInBits = MI.getOperand(2).getImm(); 1124 1125 // So long as the new type has more bits than the bits we're extending we 1126 // don't need to break it apart. 1127 if (NarrowTy.getScalarSizeInBits() >= SizeInBits) { 1128 Observer.changingInstr(MI); 1129 // We don't lose any non-extension bits by truncating the src and 1130 // sign-extending the dst. 1131 MachineOperand &MO1 = MI.getOperand(1); 1132 auto TruncMIB = MIRBuilder.buildTrunc(NarrowTy, MO1); 1133 MO1.setReg(TruncMIB.getReg(0)); 1134 1135 MachineOperand &MO2 = MI.getOperand(0); 1136 Register DstExt = MRI.createGenericVirtualRegister(NarrowTy); 1137 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); 1138 MIRBuilder.buildSExt(MO2, DstExt); 1139 MO2.setReg(DstExt); 1140 Observer.changedInstr(MI); 1141 return Legalized; 1142 } 1143 1144 // Break it apart. Components below the extension point are unmodified. The 1145 // component containing the extension point becomes a narrower SEXT_INREG. 1146 // Components above it are ashr'd from the component containing the 1147 // extension point. 1148 if (SizeOp0 % NarrowSize != 0) 1149 return UnableToLegalize; 1150 int NumParts = SizeOp0 / NarrowSize; 1151 1152 // List the registers where the destination will be scattered. 1153 SmallVector<Register, 2> DstRegs; 1154 // List the registers where the source will be split. 1155 SmallVector<Register, 2> SrcRegs; 1156 1157 // Create all the temporary registers. 1158 for (int i = 0; i < NumParts; ++i) { 1159 Register SrcReg = MRI.createGenericVirtualRegister(NarrowTy); 1160 1161 SrcRegs.push_back(SrcReg); 1162 } 1163 1164 // Explode the big arguments into smaller chunks. 1165 MIRBuilder.buildUnmerge(SrcRegs, MI.getOperand(1)); 1166 1167 Register AshrCstReg = 1168 MIRBuilder.buildConstant(NarrowTy, NarrowTy.getScalarSizeInBits() - 1) 1169 .getReg(0); 1170 Register FullExtensionReg = 0; 1171 Register PartialExtensionReg = 0; 1172 1173 // Do the operation on each small part. 1174 for (int i = 0; i < NumParts; ++i) { 1175 if ((i + 1) * NarrowTy.getScalarSizeInBits() < SizeInBits) 1176 DstRegs.push_back(SrcRegs[i]); 1177 else if (i * NarrowTy.getScalarSizeInBits() > SizeInBits) { 1178 assert(PartialExtensionReg && 1179 "Expected to visit partial extension before full"); 1180 if (FullExtensionReg) { 1181 DstRegs.push_back(FullExtensionReg); 1182 continue; 1183 } 1184 DstRegs.push_back( 1185 MIRBuilder.buildAShr(NarrowTy, PartialExtensionReg, AshrCstReg) 1186 .getReg(0)); 1187 FullExtensionReg = DstRegs.back(); 1188 } else { 1189 DstRegs.push_back( 1190 MIRBuilder 1191 .buildInstr( 1192 TargetOpcode::G_SEXT_INREG, {NarrowTy}, 1193 {SrcRegs[i], SizeInBits % NarrowTy.getScalarSizeInBits()}) 1194 .getReg(0)); 1195 PartialExtensionReg = DstRegs.back(); 1196 } 1197 } 1198 1199 // Gather the destination registers into the final destination. 1200 Register DstReg = MI.getOperand(0).getReg(); 1201 MIRBuilder.buildMerge(DstReg, DstRegs); 1202 MI.eraseFromParent(); 1203 return Legalized; 1204 } 1205 case TargetOpcode::G_BSWAP: 1206 case TargetOpcode::G_BITREVERSE: { 1207 if (SizeOp0 % NarrowSize != 0) 1208 return UnableToLegalize; 1209 1210 Observer.changingInstr(MI); 1211 SmallVector<Register, 2> SrcRegs, DstRegs; 1212 unsigned NumParts = SizeOp0 / NarrowSize; 1213 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs); 1214 1215 for (unsigned i = 0; i < NumParts; ++i) { 1216 auto DstPart = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy}, 1217 {SrcRegs[NumParts - 1 - i]}); 1218 DstRegs.push_back(DstPart.getReg(0)); 1219 } 1220 1221 MIRBuilder.buildMerge(MI.getOperand(0), DstRegs); 1222 1223 Observer.changedInstr(MI); 1224 MI.eraseFromParent(); 1225 return Legalized; 1226 } 1227 case TargetOpcode::G_PTR_ADD: 1228 case TargetOpcode::G_PTRMASK: { 1229 if (TypeIdx != 1) 1230 return UnableToLegalize; 1231 Observer.changingInstr(MI); 1232 narrowScalarSrc(MI, NarrowTy, 2); 1233 Observer.changedInstr(MI); 1234 return Legalized; 1235 } 1236 case TargetOpcode::G_FPTOUI: { 1237 if (TypeIdx != 0) 1238 return UnableToLegalize; 1239 Observer.changingInstr(MI); 1240 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT); 1241 Observer.changedInstr(MI); 1242 return Legalized; 1243 } 1244 case TargetOpcode::G_FPTOSI: { 1245 if (TypeIdx != 0) 1246 return UnableToLegalize; 1247 Observer.changingInstr(MI); 1248 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_SEXT); 1249 Observer.changedInstr(MI); 1250 return Legalized; 1251 } 1252 case TargetOpcode::G_FPEXT: 1253 if (TypeIdx != 0) 1254 return UnableToLegalize; 1255 Observer.changingInstr(MI); 1256 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_FPEXT); 1257 Observer.changedInstr(MI); 1258 return Legalized; 1259 } 1260 } 1261 1262 Register LegalizerHelper::coerceToScalar(Register Val) { 1263 LLT Ty = MRI.getType(Val); 1264 if (Ty.isScalar()) 1265 return Val; 1266 1267 const DataLayout &DL = MIRBuilder.getDataLayout(); 1268 LLT NewTy = LLT::scalar(Ty.getSizeInBits()); 1269 if (Ty.isPointer()) { 1270 if (DL.isNonIntegralAddressSpace(Ty.getAddressSpace())) 1271 return Register(); 1272 return MIRBuilder.buildPtrToInt(NewTy, Val).getReg(0); 1273 } 1274 1275 Register NewVal = Val; 1276 1277 assert(Ty.isVector()); 1278 LLT EltTy = Ty.getElementType(); 1279 if (EltTy.isPointer()) 1280 NewVal = MIRBuilder.buildPtrToInt(NewTy, NewVal).getReg(0); 1281 return MIRBuilder.buildBitcast(NewTy, NewVal).getReg(0); 1282 } 1283 1284 void LegalizerHelper::widenScalarSrc(MachineInstr &MI, LLT WideTy, 1285 unsigned OpIdx, unsigned ExtOpcode) { 1286 MachineOperand &MO = MI.getOperand(OpIdx); 1287 auto ExtB = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MO}); 1288 MO.setReg(ExtB.getReg(0)); 1289 } 1290 1291 void LegalizerHelper::narrowScalarSrc(MachineInstr &MI, LLT NarrowTy, 1292 unsigned OpIdx) { 1293 MachineOperand &MO = MI.getOperand(OpIdx); 1294 auto ExtB = MIRBuilder.buildTrunc(NarrowTy, MO); 1295 MO.setReg(ExtB.getReg(0)); 1296 } 1297 1298 void LegalizerHelper::widenScalarDst(MachineInstr &MI, LLT WideTy, 1299 unsigned OpIdx, unsigned TruncOpcode) { 1300 MachineOperand &MO = MI.getOperand(OpIdx); 1301 Register DstExt = MRI.createGenericVirtualRegister(WideTy); 1302 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); 1303 MIRBuilder.buildInstr(TruncOpcode, {MO}, {DstExt}); 1304 MO.setReg(DstExt); 1305 } 1306 1307 void LegalizerHelper::narrowScalarDst(MachineInstr &MI, LLT NarrowTy, 1308 unsigned OpIdx, unsigned ExtOpcode) { 1309 MachineOperand &MO = MI.getOperand(OpIdx); 1310 Register DstTrunc = MRI.createGenericVirtualRegister(NarrowTy); 1311 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); 1312 MIRBuilder.buildInstr(ExtOpcode, {MO}, {DstTrunc}); 1313 MO.setReg(DstTrunc); 1314 } 1315 1316 void LegalizerHelper::moreElementsVectorDst(MachineInstr &MI, LLT WideTy, 1317 unsigned OpIdx) { 1318 MachineOperand &MO = MI.getOperand(OpIdx); 1319 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); 1320 MO.setReg(widenWithUnmerge(WideTy, MO.getReg())); 1321 } 1322 1323 void LegalizerHelper::moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy, 1324 unsigned OpIdx) { 1325 MachineOperand &MO = MI.getOperand(OpIdx); 1326 1327 LLT OldTy = MRI.getType(MO.getReg()); 1328 unsigned OldElts = OldTy.getNumElements(); 1329 unsigned NewElts = MoreTy.getNumElements(); 1330 1331 unsigned NumParts = NewElts / OldElts; 1332 1333 // Use concat_vectors if the result is a multiple of the number of elements. 1334 if (NumParts * OldElts == NewElts) { 1335 SmallVector<Register, 8> Parts; 1336 Parts.push_back(MO.getReg()); 1337 1338 Register ImpDef = MIRBuilder.buildUndef(OldTy).getReg(0); 1339 for (unsigned I = 1; I != NumParts; ++I) 1340 Parts.push_back(ImpDef); 1341 1342 auto Concat = MIRBuilder.buildConcatVectors(MoreTy, Parts); 1343 MO.setReg(Concat.getReg(0)); 1344 return; 1345 } 1346 1347 Register MoreReg = MRI.createGenericVirtualRegister(MoreTy); 1348 Register ImpDef = MIRBuilder.buildUndef(MoreTy).getReg(0); 1349 MIRBuilder.buildInsert(MoreReg, ImpDef, MO.getReg(), 0); 1350 MO.setReg(MoreReg); 1351 } 1352 1353 void LegalizerHelper::bitcastSrc(MachineInstr &MI, LLT CastTy, unsigned OpIdx) { 1354 MachineOperand &Op = MI.getOperand(OpIdx); 1355 Op.setReg(MIRBuilder.buildBitcast(CastTy, Op).getReg(0)); 1356 } 1357 1358 void LegalizerHelper::bitcastDst(MachineInstr &MI, LLT CastTy, unsigned OpIdx) { 1359 MachineOperand &MO = MI.getOperand(OpIdx); 1360 Register CastDst = MRI.createGenericVirtualRegister(CastTy); 1361 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); 1362 MIRBuilder.buildBitcast(MO, CastDst); 1363 MO.setReg(CastDst); 1364 } 1365 1366 LegalizerHelper::LegalizeResult 1367 LegalizerHelper::widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx, 1368 LLT WideTy) { 1369 if (TypeIdx != 1) 1370 return UnableToLegalize; 1371 1372 Register DstReg = MI.getOperand(0).getReg(); 1373 LLT DstTy = MRI.getType(DstReg); 1374 if (DstTy.isVector()) 1375 return UnableToLegalize; 1376 1377 Register Src1 = MI.getOperand(1).getReg(); 1378 LLT SrcTy = MRI.getType(Src1); 1379 const int DstSize = DstTy.getSizeInBits(); 1380 const int SrcSize = SrcTy.getSizeInBits(); 1381 const int WideSize = WideTy.getSizeInBits(); 1382 const int NumMerge = (DstSize + WideSize - 1) / WideSize; 1383 1384 unsigned NumOps = MI.getNumOperands(); 1385 unsigned NumSrc = MI.getNumOperands() - 1; 1386 unsigned PartSize = DstTy.getSizeInBits() / NumSrc; 1387 1388 if (WideSize >= DstSize) { 1389 // Directly pack the bits in the target type. 1390 Register ResultReg = MIRBuilder.buildZExt(WideTy, Src1).getReg(0); 1391 1392 for (unsigned I = 2; I != NumOps; ++I) { 1393 const unsigned Offset = (I - 1) * PartSize; 1394 1395 Register SrcReg = MI.getOperand(I).getReg(); 1396 assert(MRI.getType(SrcReg) == LLT::scalar(PartSize)); 1397 1398 auto ZextInput = MIRBuilder.buildZExt(WideTy, SrcReg); 1399 1400 Register NextResult = I + 1 == NumOps && WideTy == DstTy ? DstReg : 1401 MRI.createGenericVirtualRegister(WideTy); 1402 1403 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, Offset); 1404 auto Shl = MIRBuilder.buildShl(WideTy, ZextInput, ShiftAmt); 1405 MIRBuilder.buildOr(NextResult, ResultReg, Shl); 1406 ResultReg = NextResult; 1407 } 1408 1409 if (WideSize > DstSize) 1410 MIRBuilder.buildTrunc(DstReg, ResultReg); 1411 else if (DstTy.isPointer()) 1412 MIRBuilder.buildIntToPtr(DstReg, ResultReg); 1413 1414 MI.eraseFromParent(); 1415 return Legalized; 1416 } 1417 1418 // Unmerge the original values to the GCD type, and recombine to the next 1419 // multiple greater than the original type. 1420 // 1421 // %3:_(s12) = G_MERGE_VALUES %0:_(s4), %1:_(s4), %2:_(s4) -> s6 1422 // %4:_(s2), %5:_(s2) = G_UNMERGE_VALUES %0 1423 // %6:_(s2), %7:_(s2) = G_UNMERGE_VALUES %1 1424 // %8:_(s2), %9:_(s2) = G_UNMERGE_VALUES %2 1425 // %10:_(s6) = G_MERGE_VALUES %4, %5, %6 1426 // %11:_(s6) = G_MERGE_VALUES %7, %8, %9 1427 // %12:_(s12) = G_MERGE_VALUES %10, %11 1428 // 1429 // Padding with undef if necessary: 1430 // 1431 // %2:_(s8) = G_MERGE_VALUES %0:_(s4), %1:_(s4) -> s6 1432 // %3:_(s2), %4:_(s2) = G_UNMERGE_VALUES %0 1433 // %5:_(s2), %6:_(s2) = G_UNMERGE_VALUES %1 1434 // %7:_(s2) = G_IMPLICIT_DEF 1435 // %8:_(s6) = G_MERGE_VALUES %3, %4, %5 1436 // %9:_(s6) = G_MERGE_VALUES %6, %7, %7 1437 // %10:_(s12) = G_MERGE_VALUES %8, %9 1438 1439 const int GCD = greatestCommonDivisor(SrcSize, WideSize); 1440 LLT GCDTy = LLT::scalar(GCD); 1441 1442 SmallVector<Register, 8> Parts; 1443 SmallVector<Register, 8> NewMergeRegs; 1444 SmallVector<Register, 8> Unmerges; 1445 LLT WideDstTy = LLT::scalar(NumMerge * WideSize); 1446 1447 // Decompose the original operands if they don't evenly divide. 1448 for (int I = 1, E = MI.getNumOperands(); I != E; ++I) { 1449 Register SrcReg = MI.getOperand(I).getReg(); 1450 if (GCD == SrcSize) { 1451 Unmerges.push_back(SrcReg); 1452 } else { 1453 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg); 1454 for (int J = 0, JE = Unmerge->getNumOperands() - 1; J != JE; ++J) 1455 Unmerges.push_back(Unmerge.getReg(J)); 1456 } 1457 } 1458 1459 // Pad with undef to the next size that is a multiple of the requested size. 1460 if (static_cast<int>(Unmerges.size()) != NumMerge * WideSize) { 1461 Register UndefReg = MIRBuilder.buildUndef(GCDTy).getReg(0); 1462 for (int I = Unmerges.size(); I != NumMerge * WideSize; ++I) 1463 Unmerges.push_back(UndefReg); 1464 } 1465 1466 const int PartsPerGCD = WideSize / GCD; 1467 1468 // Build merges of each piece. 1469 ArrayRef<Register> Slicer(Unmerges); 1470 for (int I = 0; I != NumMerge; ++I, Slicer = Slicer.drop_front(PartsPerGCD)) { 1471 auto Merge = MIRBuilder.buildMerge(WideTy, Slicer.take_front(PartsPerGCD)); 1472 NewMergeRegs.push_back(Merge.getReg(0)); 1473 } 1474 1475 // A truncate may be necessary if the requested type doesn't evenly divide the 1476 // original result type. 1477 if (DstTy.getSizeInBits() == WideDstTy.getSizeInBits()) { 1478 MIRBuilder.buildMerge(DstReg, NewMergeRegs); 1479 } else { 1480 auto FinalMerge = MIRBuilder.buildMerge(WideDstTy, NewMergeRegs); 1481 MIRBuilder.buildTrunc(DstReg, FinalMerge.getReg(0)); 1482 } 1483 1484 MI.eraseFromParent(); 1485 return Legalized; 1486 } 1487 1488 Register LegalizerHelper::widenWithUnmerge(LLT WideTy, Register OrigReg) { 1489 Register WideReg = MRI.createGenericVirtualRegister(WideTy); 1490 LLT OrigTy = MRI.getType(OrigReg); 1491 LLT LCMTy = getLCMType(WideTy, OrigTy); 1492 1493 const int NumMergeParts = LCMTy.getSizeInBits() / WideTy.getSizeInBits(); 1494 const int NumUnmergeParts = LCMTy.getSizeInBits() / OrigTy.getSizeInBits(); 1495 1496 Register UnmergeSrc = WideReg; 1497 1498 // Create a merge to the LCM type, padding with undef 1499 // %0:_(<3 x s32>) = G_FOO => <4 x s32> 1500 // => 1501 // %1:_(<4 x s32>) = G_FOO 1502 // %2:_(<4 x s32>) = G_IMPLICIT_DEF 1503 // %3:_(<12 x s32>) = G_CONCAT_VECTORS %1, %2, %2 1504 // %0:_(<3 x s32>), %4:_, %5:_, %6:_ = G_UNMERGE_VALUES %3 1505 if (NumMergeParts > 1) { 1506 Register Undef = MIRBuilder.buildUndef(WideTy).getReg(0); 1507 SmallVector<Register, 8> MergeParts(NumMergeParts, Undef); 1508 MergeParts[0] = WideReg; 1509 UnmergeSrc = MIRBuilder.buildMerge(LCMTy, MergeParts).getReg(0); 1510 } 1511 1512 // Unmerge to the original register and pad with dead defs. 1513 SmallVector<Register, 8> UnmergeResults(NumUnmergeParts); 1514 UnmergeResults[0] = OrigReg; 1515 for (int I = 1; I != NumUnmergeParts; ++I) 1516 UnmergeResults[I] = MRI.createGenericVirtualRegister(OrigTy); 1517 1518 MIRBuilder.buildUnmerge(UnmergeResults, UnmergeSrc); 1519 return WideReg; 1520 } 1521 1522 LegalizerHelper::LegalizeResult 1523 LegalizerHelper::widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx, 1524 LLT WideTy) { 1525 if (TypeIdx != 0) 1526 return UnableToLegalize; 1527 1528 int NumDst = MI.getNumOperands() - 1; 1529 Register SrcReg = MI.getOperand(NumDst).getReg(); 1530 LLT SrcTy = MRI.getType(SrcReg); 1531 if (SrcTy.isVector()) 1532 return UnableToLegalize; 1533 1534 Register Dst0Reg = MI.getOperand(0).getReg(); 1535 LLT DstTy = MRI.getType(Dst0Reg); 1536 if (!DstTy.isScalar()) 1537 return UnableToLegalize; 1538 1539 if (WideTy.getSizeInBits() >= SrcTy.getSizeInBits()) { 1540 if (SrcTy.isPointer()) { 1541 const DataLayout &DL = MIRBuilder.getDataLayout(); 1542 if (DL.isNonIntegralAddressSpace(SrcTy.getAddressSpace())) { 1543 LLVM_DEBUG( 1544 dbgs() << "Not casting non-integral address space integer\n"); 1545 return UnableToLegalize; 1546 } 1547 1548 SrcTy = LLT::scalar(SrcTy.getSizeInBits()); 1549 SrcReg = MIRBuilder.buildPtrToInt(SrcTy, SrcReg).getReg(0); 1550 } 1551 1552 // Widen SrcTy to WideTy. This does not affect the result, but since the 1553 // user requested this size, it is probably better handled than SrcTy and 1554 // should reduce the total number of legalization artifacts 1555 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) { 1556 SrcTy = WideTy; 1557 SrcReg = MIRBuilder.buildAnyExt(WideTy, SrcReg).getReg(0); 1558 } 1559 1560 // Theres no unmerge type to target. Directly extract the bits from the 1561 // source type 1562 unsigned DstSize = DstTy.getSizeInBits(); 1563 1564 MIRBuilder.buildTrunc(Dst0Reg, SrcReg); 1565 for (int I = 1; I != NumDst; ++I) { 1566 auto ShiftAmt = MIRBuilder.buildConstant(SrcTy, DstSize * I); 1567 auto Shr = MIRBuilder.buildLShr(SrcTy, SrcReg, ShiftAmt); 1568 MIRBuilder.buildTrunc(MI.getOperand(I), Shr); 1569 } 1570 1571 MI.eraseFromParent(); 1572 return Legalized; 1573 } 1574 1575 // Extend the source to a wider type. 1576 LLT LCMTy = getLCMType(SrcTy, WideTy); 1577 1578 Register WideSrc = SrcReg; 1579 if (LCMTy.getSizeInBits() != SrcTy.getSizeInBits()) { 1580 // TODO: If this is an integral address space, cast to integer and anyext. 1581 if (SrcTy.isPointer()) { 1582 LLVM_DEBUG(dbgs() << "Widening pointer source types not implemented\n"); 1583 return UnableToLegalize; 1584 } 1585 1586 WideSrc = MIRBuilder.buildAnyExt(LCMTy, WideSrc).getReg(0); 1587 } 1588 1589 auto Unmerge = MIRBuilder.buildUnmerge(WideTy, WideSrc); 1590 1591 // Create a sequence of unmerges to the original results. since we may have 1592 // widened the source, we will need to pad the results with dead defs to cover 1593 // the source register. 1594 // e.g. widen s16 to s32: 1595 // %1:_(s16), %2:_(s16), %3:_(s16) = G_UNMERGE_VALUES %0:_(s48) 1596 // 1597 // => 1598 // %4:_(s64) = G_ANYEXT %0:_(s48) 1599 // %5:_(s32), %6:_(s32) = G_UNMERGE_VALUES %4 ; Requested unmerge 1600 // %1:_(s16), %2:_(s16) = G_UNMERGE_VALUES %5 ; unpack to original regs 1601 // %3:_(s16), dead %7 = G_UNMERGE_VALUES %6 ; original reg + extra dead def 1602 1603 const int NumUnmerge = Unmerge->getNumOperands() - 1; 1604 const int PartsPerUnmerge = WideTy.getSizeInBits() / DstTy.getSizeInBits(); 1605 1606 for (int I = 0; I != NumUnmerge; ++I) { 1607 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES); 1608 1609 for (int J = 0; J != PartsPerUnmerge; ++J) { 1610 int Idx = I * PartsPerUnmerge + J; 1611 if (Idx < NumDst) 1612 MIB.addDef(MI.getOperand(Idx).getReg()); 1613 else { 1614 // Create dead def for excess components. 1615 MIB.addDef(MRI.createGenericVirtualRegister(DstTy)); 1616 } 1617 } 1618 1619 MIB.addUse(Unmerge.getReg(I)); 1620 } 1621 1622 MI.eraseFromParent(); 1623 return Legalized; 1624 } 1625 1626 LegalizerHelper::LegalizeResult 1627 LegalizerHelper::widenScalarExtract(MachineInstr &MI, unsigned TypeIdx, 1628 LLT WideTy) { 1629 Register DstReg = MI.getOperand(0).getReg(); 1630 Register SrcReg = MI.getOperand(1).getReg(); 1631 LLT SrcTy = MRI.getType(SrcReg); 1632 1633 LLT DstTy = MRI.getType(DstReg); 1634 unsigned Offset = MI.getOperand(2).getImm(); 1635 1636 if (TypeIdx == 0) { 1637 if (SrcTy.isVector() || DstTy.isVector()) 1638 return UnableToLegalize; 1639 1640 SrcOp Src(SrcReg); 1641 if (SrcTy.isPointer()) { 1642 // Extracts from pointers can be handled only if they are really just 1643 // simple integers. 1644 const DataLayout &DL = MIRBuilder.getDataLayout(); 1645 if (DL.isNonIntegralAddressSpace(SrcTy.getAddressSpace())) 1646 return UnableToLegalize; 1647 1648 LLT SrcAsIntTy = LLT::scalar(SrcTy.getSizeInBits()); 1649 Src = MIRBuilder.buildPtrToInt(SrcAsIntTy, Src); 1650 SrcTy = SrcAsIntTy; 1651 } 1652 1653 if (DstTy.isPointer()) 1654 return UnableToLegalize; 1655 1656 if (Offset == 0) { 1657 // Avoid a shift in the degenerate case. 1658 MIRBuilder.buildTrunc(DstReg, 1659 MIRBuilder.buildAnyExtOrTrunc(WideTy, Src)); 1660 MI.eraseFromParent(); 1661 return Legalized; 1662 } 1663 1664 // Do a shift in the source type. 1665 LLT ShiftTy = SrcTy; 1666 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) { 1667 Src = MIRBuilder.buildAnyExt(WideTy, Src); 1668 ShiftTy = WideTy; 1669 } 1670 1671 auto LShr = MIRBuilder.buildLShr( 1672 ShiftTy, Src, MIRBuilder.buildConstant(ShiftTy, Offset)); 1673 MIRBuilder.buildTrunc(DstReg, LShr); 1674 MI.eraseFromParent(); 1675 return Legalized; 1676 } 1677 1678 if (SrcTy.isScalar()) { 1679 Observer.changingInstr(MI); 1680 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); 1681 Observer.changedInstr(MI); 1682 return Legalized; 1683 } 1684 1685 if (!SrcTy.isVector()) 1686 return UnableToLegalize; 1687 1688 if (DstTy != SrcTy.getElementType()) 1689 return UnableToLegalize; 1690 1691 if (Offset % SrcTy.getScalarSizeInBits() != 0) 1692 return UnableToLegalize; 1693 1694 Observer.changingInstr(MI); 1695 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); 1696 1697 MI.getOperand(2).setImm((WideTy.getSizeInBits() / SrcTy.getSizeInBits()) * 1698 Offset); 1699 widenScalarDst(MI, WideTy.getScalarType(), 0); 1700 Observer.changedInstr(MI); 1701 return Legalized; 1702 } 1703 1704 LegalizerHelper::LegalizeResult 1705 LegalizerHelper::widenScalarInsert(MachineInstr &MI, unsigned TypeIdx, 1706 LLT WideTy) { 1707 if (TypeIdx != 0 || WideTy.isVector()) 1708 return UnableToLegalize; 1709 Observer.changingInstr(MI); 1710 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); 1711 widenScalarDst(MI, WideTy); 1712 Observer.changedInstr(MI); 1713 return Legalized; 1714 } 1715 1716 LegalizerHelper::LegalizeResult 1717 LegalizerHelper::widenScalarAddSubSat(MachineInstr &MI, unsigned TypeIdx, 1718 LLT WideTy) { 1719 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SADDSAT || 1720 MI.getOpcode() == TargetOpcode::G_SSUBSAT; 1721 // We can convert this to: 1722 // 1. Any extend iN to iM 1723 // 2. SHL by M-N 1724 // 3. [US][ADD|SUB]SAT 1725 // 4. L/ASHR by M-N 1726 // 1727 // It may be more efficient to lower this to a min and a max operation in 1728 // the higher precision arithmetic if the promoted operation isn't legal, 1729 // but this decision is up to the target's lowering request. 1730 Register DstReg = MI.getOperand(0).getReg(); 1731 1732 unsigned NewBits = WideTy.getScalarSizeInBits(); 1733 unsigned SHLAmount = NewBits - MRI.getType(DstReg).getScalarSizeInBits(); 1734 1735 auto LHS = MIRBuilder.buildAnyExt(WideTy, MI.getOperand(1)); 1736 auto RHS = MIRBuilder.buildAnyExt(WideTy, MI.getOperand(2)); 1737 auto ShiftK = MIRBuilder.buildConstant(WideTy, SHLAmount); 1738 auto ShiftL = MIRBuilder.buildShl(WideTy, LHS, ShiftK); 1739 auto ShiftR = MIRBuilder.buildShl(WideTy, RHS, ShiftK); 1740 1741 auto WideInst = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy}, 1742 {ShiftL, ShiftR}, MI.getFlags()); 1743 1744 // Use a shift that will preserve the number of sign bits when the trunc is 1745 // folded away. 1746 auto Result = IsSigned ? MIRBuilder.buildAShr(WideTy, WideInst, ShiftK) 1747 : MIRBuilder.buildLShr(WideTy, WideInst, ShiftK); 1748 1749 MIRBuilder.buildTrunc(DstReg, Result); 1750 MI.eraseFromParent(); 1751 return Legalized; 1752 } 1753 1754 LegalizerHelper::LegalizeResult 1755 LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { 1756 switch (MI.getOpcode()) { 1757 default: 1758 return UnableToLegalize; 1759 case TargetOpcode::G_EXTRACT: 1760 return widenScalarExtract(MI, TypeIdx, WideTy); 1761 case TargetOpcode::G_INSERT: 1762 return widenScalarInsert(MI, TypeIdx, WideTy); 1763 case TargetOpcode::G_MERGE_VALUES: 1764 return widenScalarMergeValues(MI, TypeIdx, WideTy); 1765 case TargetOpcode::G_UNMERGE_VALUES: 1766 return widenScalarUnmergeValues(MI, TypeIdx, WideTy); 1767 case TargetOpcode::G_UADDO: 1768 case TargetOpcode::G_USUBO: { 1769 if (TypeIdx == 1) 1770 return UnableToLegalize; // TODO 1771 auto LHSZext = MIRBuilder.buildZExt(WideTy, MI.getOperand(2)); 1772 auto RHSZext = MIRBuilder.buildZExt(WideTy, MI.getOperand(3)); 1773 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_UADDO 1774 ? TargetOpcode::G_ADD 1775 : TargetOpcode::G_SUB; 1776 // Do the arithmetic in the larger type. 1777 auto NewOp = MIRBuilder.buildInstr(Opcode, {WideTy}, {LHSZext, RHSZext}); 1778 LLT OrigTy = MRI.getType(MI.getOperand(0).getReg()); 1779 APInt Mask = 1780 APInt::getLowBitsSet(WideTy.getSizeInBits(), OrigTy.getSizeInBits()); 1781 auto AndOp = MIRBuilder.buildAnd( 1782 WideTy, NewOp, MIRBuilder.buildConstant(WideTy, Mask)); 1783 // There is no overflow if the AndOp is the same as NewOp. 1784 MIRBuilder.buildICmp(CmpInst::ICMP_NE, MI.getOperand(1), NewOp, AndOp); 1785 // Now trunc the NewOp to the original result. 1786 MIRBuilder.buildTrunc(MI.getOperand(0), NewOp); 1787 MI.eraseFromParent(); 1788 return Legalized; 1789 } 1790 case TargetOpcode::G_SADDSAT: 1791 case TargetOpcode::G_SSUBSAT: 1792 case TargetOpcode::G_UADDSAT: 1793 case TargetOpcode::G_USUBSAT: 1794 return widenScalarAddSubSat(MI, TypeIdx, WideTy); 1795 case TargetOpcode::G_CTTZ: 1796 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 1797 case TargetOpcode::G_CTLZ: 1798 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 1799 case TargetOpcode::G_CTPOP: { 1800 if (TypeIdx == 0) { 1801 Observer.changingInstr(MI); 1802 widenScalarDst(MI, WideTy, 0); 1803 Observer.changedInstr(MI); 1804 return Legalized; 1805 } 1806 1807 Register SrcReg = MI.getOperand(1).getReg(); 1808 1809 // First ZEXT the input. 1810 auto MIBSrc = MIRBuilder.buildZExt(WideTy, SrcReg); 1811 LLT CurTy = MRI.getType(SrcReg); 1812 if (MI.getOpcode() == TargetOpcode::G_CTTZ) { 1813 // The count is the same in the larger type except if the original 1814 // value was zero. This can be handled by setting the bit just off 1815 // the top of the original type. 1816 auto TopBit = 1817 APInt::getOneBitSet(WideTy.getSizeInBits(), CurTy.getSizeInBits()); 1818 MIBSrc = MIRBuilder.buildOr( 1819 WideTy, MIBSrc, MIRBuilder.buildConstant(WideTy, TopBit)); 1820 } 1821 1822 // Perform the operation at the larger size. 1823 auto MIBNewOp = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy}, {MIBSrc}); 1824 // This is already the correct result for CTPOP and CTTZs 1825 if (MI.getOpcode() == TargetOpcode::G_CTLZ || 1826 MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF) { 1827 // The correct result is NewOp - (Difference in widety and current ty). 1828 unsigned SizeDiff = WideTy.getSizeInBits() - CurTy.getSizeInBits(); 1829 MIBNewOp = MIRBuilder.buildSub( 1830 WideTy, MIBNewOp, MIRBuilder.buildConstant(WideTy, SizeDiff)); 1831 } 1832 1833 MIRBuilder.buildZExtOrTrunc(MI.getOperand(0), MIBNewOp); 1834 MI.eraseFromParent(); 1835 return Legalized; 1836 } 1837 case TargetOpcode::G_BSWAP: { 1838 Observer.changingInstr(MI); 1839 Register DstReg = MI.getOperand(0).getReg(); 1840 1841 Register ShrReg = MRI.createGenericVirtualRegister(WideTy); 1842 Register DstExt = MRI.createGenericVirtualRegister(WideTy); 1843 Register ShiftAmtReg = MRI.createGenericVirtualRegister(WideTy); 1844 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); 1845 1846 MI.getOperand(0).setReg(DstExt); 1847 1848 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); 1849 1850 LLT Ty = MRI.getType(DstReg); 1851 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits(); 1852 MIRBuilder.buildConstant(ShiftAmtReg, DiffBits); 1853 MIRBuilder.buildLShr(ShrReg, DstExt, ShiftAmtReg); 1854 1855 MIRBuilder.buildTrunc(DstReg, ShrReg); 1856 Observer.changedInstr(MI); 1857 return Legalized; 1858 } 1859 case TargetOpcode::G_BITREVERSE: { 1860 Observer.changingInstr(MI); 1861 1862 Register DstReg = MI.getOperand(0).getReg(); 1863 LLT Ty = MRI.getType(DstReg); 1864 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits(); 1865 1866 Register DstExt = MRI.createGenericVirtualRegister(WideTy); 1867 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); 1868 MI.getOperand(0).setReg(DstExt); 1869 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); 1870 1871 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, DiffBits); 1872 auto Shift = MIRBuilder.buildLShr(WideTy, DstExt, ShiftAmt); 1873 MIRBuilder.buildTrunc(DstReg, Shift); 1874 Observer.changedInstr(MI); 1875 return Legalized; 1876 } 1877 case TargetOpcode::G_FREEZE: 1878 Observer.changingInstr(MI); 1879 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); 1880 widenScalarDst(MI, WideTy); 1881 Observer.changedInstr(MI); 1882 return Legalized; 1883 1884 case TargetOpcode::G_ADD: 1885 case TargetOpcode::G_AND: 1886 case TargetOpcode::G_MUL: 1887 case TargetOpcode::G_OR: 1888 case TargetOpcode::G_XOR: 1889 case TargetOpcode::G_SUB: 1890 // Perform operation at larger width (any extension is fines here, high bits 1891 // don't affect the result) and then truncate the result back to the 1892 // original type. 1893 Observer.changingInstr(MI); 1894 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); 1895 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT); 1896 widenScalarDst(MI, WideTy); 1897 Observer.changedInstr(MI); 1898 return Legalized; 1899 1900 case TargetOpcode::G_SHL: 1901 Observer.changingInstr(MI); 1902 1903 if (TypeIdx == 0) { 1904 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); 1905 widenScalarDst(MI, WideTy); 1906 } else { 1907 assert(TypeIdx == 1); 1908 // The "number of bits to shift" operand must preserve its value as an 1909 // unsigned integer: 1910 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); 1911 } 1912 1913 Observer.changedInstr(MI); 1914 return Legalized; 1915 1916 case TargetOpcode::G_SDIV: 1917 case TargetOpcode::G_SREM: 1918 case TargetOpcode::G_SMIN: 1919 case TargetOpcode::G_SMAX: 1920 Observer.changingInstr(MI); 1921 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT); 1922 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT); 1923 widenScalarDst(MI, WideTy); 1924 Observer.changedInstr(MI); 1925 return Legalized; 1926 1927 case TargetOpcode::G_ASHR: 1928 case TargetOpcode::G_LSHR: 1929 Observer.changingInstr(MI); 1930 1931 if (TypeIdx == 0) { 1932 unsigned CvtOp = MI.getOpcode() == TargetOpcode::G_ASHR ? 1933 TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT; 1934 1935 widenScalarSrc(MI, WideTy, 1, CvtOp); 1936 widenScalarDst(MI, WideTy); 1937 } else { 1938 assert(TypeIdx == 1); 1939 // The "number of bits to shift" operand must preserve its value as an 1940 // unsigned integer: 1941 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); 1942 } 1943 1944 Observer.changedInstr(MI); 1945 return Legalized; 1946 case TargetOpcode::G_UDIV: 1947 case TargetOpcode::G_UREM: 1948 case TargetOpcode::G_UMIN: 1949 case TargetOpcode::G_UMAX: 1950 Observer.changingInstr(MI); 1951 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT); 1952 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); 1953 widenScalarDst(MI, WideTy); 1954 Observer.changedInstr(MI); 1955 return Legalized; 1956 1957 case TargetOpcode::G_SELECT: 1958 Observer.changingInstr(MI); 1959 if (TypeIdx == 0) { 1960 // Perform operation at larger width (any extension is fine here, high 1961 // bits don't affect the result) and then truncate the result back to the 1962 // original type. 1963 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT); 1964 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT); 1965 widenScalarDst(MI, WideTy); 1966 } else { 1967 bool IsVec = MRI.getType(MI.getOperand(1).getReg()).isVector(); 1968 // Explicit extension is required here since high bits affect the result. 1969 widenScalarSrc(MI, WideTy, 1, MIRBuilder.getBoolExtOp(IsVec, false)); 1970 } 1971 Observer.changedInstr(MI); 1972 return Legalized; 1973 1974 case TargetOpcode::G_FPTOSI: 1975 case TargetOpcode::G_FPTOUI: 1976 Observer.changingInstr(MI); 1977 1978 if (TypeIdx == 0) 1979 widenScalarDst(MI, WideTy); 1980 else 1981 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT); 1982 1983 Observer.changedInstr(MI); 1984 return Legalized; 1985 case TargetOpcode::G_SITOFP: 1986 Observer.changingInstr(MI); 1987 1988 if (TypeIdx == 0) 1989 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC); 1990 else 1991 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT); 1992 1993 Observer.changedInstr(MI); 1994 return Legalized; 1995 case TargetOpcode::G_UITOFP: 1996 Observer.changingInstr(MI); 1997 1998 if (TypeIdx == 0) 1999 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC); 2000 else 2001 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT); 2002 2003 Observer.changedInstr(MI); 2004 return Legalized; 2005 case TargetOpcode::G_LOAD: 2006 case TargetOpcode::G_SEXTLOAD: 2007 case TargetOpcode::G_ZEXTLOAD: 2008 Observer.changingInstr(MI); 2009 widenScalarDst(MI, WideTy); 2010 Observer.changedInstr(MI); 2011 return Legalized; 2012 2013 case TargetOpcode::G_STORE: { 2014 if (TypeIdx != 0) 2015 return UnableToLegalize; 2016 2017 LLT Ty = MRI.getType(MI.getOperand(0).getReg()); 2018 if (!isPowerOf2_32(Ty.getSizeInBits())) 2019 return UnableToLegalize; 2020 2021 Observer.changingInstr(MI); 2022 2023 unsigned ExtType = Ty.getScalarSizeInBits() == 1 ? 2024 TargetOpcode::G_ZEXT : TargetOpcode::G_ANYEXT; 2025 widenScalarSrc(MI, WideTy, 0, ExtType); 2026 2027 Observer.changedInstr(MI); 2028 return Legalized; 2029 } 2030 case TargetOpcode::G_CONSTANT: { 2031 MachineOperand &SrcMO = MI.getOperand(1); 2032 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext(); 2033 unsigned ExtOpc = LI.getExtOpcodeForWideningConstant( 2034 MRI.getType(MI.getOperand(0).getReg())); 2035 assert((ExtOpc == TargetOpcode::G_ZEXT || ExtOpc == TargetOpcode::G_SEXT || 2036 ExtOpc == TargetOpcode::G_ANYEXT) && 2037 "Illegal Extend"); 2038 const APInt &SrcVal = SrcMO.getCImm()->getValue(); 2039 const APInt &Val = (ExtOpc == TargetOpcode::G_SEXT) 2040 ? SrcVal.sext(WideTy.getSizeInBits()) 2041 : SrcVal.zext(WideTy.getSizeInBits()); 2042 Observer.changingInstr(MI); 2043 SrcMO.setCImm(ConstantInt::get(Ctx, Val)); 2044 2045 widenScalarDst(MI, WideTy); 2046 Observer.changedInstr(MI); 2047 return Legalized; 2048 } 2049 case TargetOpcode::G_FCONSTANT: { 2050 MachineOperand &SrcMO = MI.getOperand(1); 2051 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext(); 2052 APFloat Val = SrcMO.getFPImm()->getValueAPF(); 2053 bool LosesInfo; 2054 switch (WideTy.getSizeInBits()) { 2055 case 32: 2056 Val.convert(APFloat::IEEEsingle(), APFloat::rmNearestTiesToEven, 2057 &LosesInfo); 2058 break; 2059 case 64: 2060 Val.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven, 2061 &LosesInfo); 2062 break; 2063 default: 2064 return UnableToLegalize; 2065 } 2066 2067 assert(!LosesInfo && "extend should always be lossless"); 2068 2069 Observer.changingInstr(MI); 2070 SrcMO.setFPImm(ConstantFP::get(Ctx, Val)); 2071 2072 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC); 2073 Observer.changedInstr(MI); 2074 return Legalized; 2075 } 2076 case TargetOpcode::G_IMPLICIT_DEF: { 2077 Observer.changingInstr(MI); 2078 widenScalarDst(MI, WideTy); 2079 Observer.changedInstr(MI); 2080 return Legalized; 2081 } 2082 case TargetOpcode::G_BRCOND: 2083 Observer.changingInstr(MI); 2084 widenScalarSrc(MI, WideTy, 0, MIRBuilder.getBoolExtOp(false, false)); 2085 Observer.changedInstr(MI); 2086 return Legalized; 2087 2088 case TargetOpcode::G_FCMP: 2089 Observer.changingInstr(MI); 2090 if (TypeIdx == 0) 2091 widenScalarDst(MI, WideTy); 2092 else { 2093 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT); 2094 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_FPEXT); 2095 } 2096 Observer.changedInstr(MI); 2097 return Legalized; 2098 2099 case TargetOpcode::G_ICMP: 2100 Observer.changingInstr(MI); 2101 if (TypeIdx == 0) 2102 widenScalarDst(MI, WideTy); 2103 else { 2104 unsigned ExtOpcode = CmpInst::isSigned(static_cast<CmpInst::Predicate>( 2105 MI.getOperand(1).getPredicate())) 2106 ? TargetOpcode::G_SEXT 2107 : TargetOpcode::G_ZEXT; 2108 widenScalarSrc(MI, WideTy, 2, ExtOpcode); 2109 widenScalarSrc(MI, WideTy, 3, ExtOpcode); 2110 } 2111 Observer.changedInstr(MI); 2112 return Legalized; 2113 2114 case TargetOpcode::G_PTR_ADD: 2115 assert(TypeIdx == 1 && "unable to legalize pointer of G_PTR_ADD"); 2116 Observer.changingInstr(MI); 2117 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT); 2118 Observer.changedInstr(MI); 2119 return Legalized; 2120 2121 case TargetOpcode::G_PHI: { 2122 assert(TypeIdx == 0 && "Expecting only Idx 0"); 2123 2124 Observer.changingInstr(MI); 2125 for (unsigned I = 1; I < MI.getNumOperands(); I += 2) { 2126 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB(); 2127 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator()); 2128 widenScalarSrc(MI, WideTy, I, TargetOpcode::G_ANYEXT); 2129 } 2130 2131 MachineBasicBlock &MBB = *MI.getParent(); 2132 MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI()); 2133 widenScalarDst(MI, WideTy); 2134 Observer.changedInstr(MI); 2135 return Legalized; 2136 } 2137 case TargetOpcode::G_EXTRACT_VECTOR_ELT: { 2138 if (TypeIdx == 0) { 2139 Register VecReg = MI.getOperand(1).getReg(); 2140 LLT VecTy = MRI.getType(VecReg); 2141 Observer.changingInstr(MI); 2142 2143 widenScalarSrc(MI, LLT::vector(VecTy.getNumElements(), 2144 WideTy.getSizeInBits()), 2145 1, TargetOpcode::G_SEXT); 2146 2147 widenScalarDst(MI, WideTy, 0); 2148 Observer.changedInstr(MI); 2149 return Legalized; 2150 } 2151 2152 if (TypeIdx != 2) 2153 return UnableToLegalize; 2154 Observer.changingInstr(MI); 2155 // TODO: Probably should be zext 2156 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT); 2157 Observer.changedInstr(MI); 2158 return Legalized; 2159 } 2160 case TargetOpcode::G_INSERT_VECTOR_ELT: { 2161 if (TypeIdx == 1) { 2162 Observer.changingInstr(MI); 2163 2164 Register VecReg = MI.getOperand(1).getReg(); 2165 LLT VecTy = MRI.getType(VecReg); 2166 LLT WideVecTy = LLT::vector(VecTy.getNumElements(), WideTy); 2167 2168 widenScalarSrc(MI, WideVecTy, 1, TargetOpcode::G_ANYEXT); 2169 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT); 2170 widenScalarDst(MI, WideVecTy, 0); 2171 Observer.changedInstr(MI); 2172 return Legalized; 2173 } 2174 2175 if (TypeIdx == 2) { 2176 Observer.changingInstr(MI); 2177 // TODO: Probably should be zext 2178 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_SEXT); 2179 Observer.changedInstr(MI); 2180 return Legalized; 2181 } 2182 2183 return UnableToLegalize; 2184 } 2185 case TargetOpcode::G_FADD: 2186 case TargetOpcode::G_FMUL: 2187 case TargetOpcode::G_FSUB: 2188 case TargetOpcode::G_FMA: 2189 case TargetOpcode::G_FMAD: 2190 case TargetOpcode::G_FNEG: 2191 case TargetOpcode::G_FABS: 2192 case TargetOpcode::G_FCANONICALIZE: 2193 case TargetOpcode::G_FMINNUM: 2194 case TargetOpcode::G_FMAXNUM: 2195 case TargetOpcode::G_FMINNUM_IEEE: 2196 case TargetOpcode::G_FMAXNUM_IEEE: 2197 case TargetOpcode::G_FMINIMUM: 2198 case TargetOpcode::G_FMAXIMUM: 2199 case TargetOpcode::G_FDIV: 2200 case TargetOpcode::G_FREM: 2201 case TargetOpcode::G_FCEIL: 2202 case TargetOpcode::G_FFLOOR: 2203 case TargetOpcode::G_FCOS: 2204 case TargetOpcode::G_FSIN: 2205 case TargetOpcode::G_FLOG10: 2206 case TargetOpcode::G_FLOG: 2207 case TargetOpcode::G_FLOG2: 2208 case TargetOpcode::G_FRINT: 2209 case TargetOpcode::G_FNEARBYINT: 2210 case TargetOpcode::G_FSQRT: 2211 case TargetOpcode::G_FEXP: 2212 case TargetOpcode::G_FEXP2: 2213 case TargetOpcode::G_FPOW: 2214 case TargetOpcode::G_INTRINSIC_TRUNC: 2215 case TargetOpcode::G_INTRINSIC_ROUND: 2216 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: 2217 assert(TypeIdx == 0); 2218 Observer.changingInstr(MI); 2219 2220 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) 2221 widenScalarSrc(MI, WideTy, I, TargetOpcode::G_FPEXT); 2222 2223 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC); 2224 Observer.changedInstr(MI); 2225 return Legalized; 2226 case TargetOpcode::G_FPOWI: { 2227 if (TypeIdx != 0) 2228 return UnableToLegalize; 2229 Observer.changingInstr(MI); 2230 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT); 2231 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC); 2232 Observer.changedInstr(MI); 2233 return Legalized; 2234 } 2235 case TargetOpcode::G_INTTOPTR: 2236 if (TypeIdx != 1) 2237 return UnableToLegalize; 2238 2239 Observer.changingInstr(MI); 2240 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT); 2241 Observer.changedInstr(MI); 2242 return Legalized; 2243 case TargetOpcode::G_PTRTOINT: 2244 if (TypeIdx != 0) 2245 return UnableToLegalize; 2246 2247 Observer.changingInstr(MI); 2248 widenScalarDst(MI, WideTy, 0); 2249 Observer.changedInstr(MI); 2250 return Legalized; 2251 case TargetOpcode::G_BUILD_VECTOR: { 2252 Observer.changingInstr(MI); 2253 2254 const LLT WideEltTy = TypeIdx == 1 ? WideTy : WideTy.getElementType(); 2255 for (int I = 1, E = MI.getNumOperands(); I != E; ++I) 2256 widenScalarSrc(MI, WideEltTy, I, TargetOpcode::G_ANYEXT); 2257 2258 // Avoid changing the result vector type if the source element type was 2259 // requested. 2260 if (TypeIdx == 1) { 2261 MI.setDesc(MIRBuilder.getTII().get(TargetOpcode::G_BUILD_VECTOR_TRUNC)); 2262 } else { 2263 widenScalarDst(MI, WideTy, 0); 2264 } 2265 2266 Observer.changedInstr(MI); 2267 return Legalized; 2268 } 2269 case TargetOpcode::G_SEXT_INREG: 2270 if (TypeIdx != 0) 2271 return UnableToLegalize; 2272 2273 Observer.changingInstr(MI); 2274 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); 2275 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_TRUNC); 2276 Observer.changedInstr(MI); 2277 return Legalized; 2278 case TargetOpcode::G_PTRMASK: { 2279 if (TypeIdx != 1) 2280 return UnableToLegalize; 2281 Observer.changingInstr(MI); 2282 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); 2283 Observer.changedInstr(MI); 2284 return Legalized; 2285 } 2286 } 2287 } 2288 2289 static void getUnmergePieces(SmallVectorImpl<Register> &Pieces, 2290 MachineIRBuilder &B, Register Src, LLT Ty) { 2291 auto Unmerge = B.buildUnmerge(Ty, Src); 2292 for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I) 2293 Pieces.push_back(Unmerge.getReg(I)); 2294 } 2295 2296 LegalizerHelper::LegalizeResult 2297 LegalizerHelper::lowerBitcast(MachineInstr &MI) { 2298 Register Dst = MI.getOperand(0).getReg(); 2299 Register Src = MI.getOperand(1).getReg(); 2300 LLT DstTy = MRI.getType(Dst); 2301 LLT SrcTy = MRI.getType(Src); 2302 2303 if (SrcTy.isVector()) { 2304 LLT SrcEltTy = SrcTy.getElementType(); 2305 SmallVector<Register, 8> SrcRegs; 2306 2307 if (DstTy.isVector()) { 2308 int NumDstElt = DstTy.getNumElements(); 2309 int NumSrcElt = SrcTy.getNumElements(); 2310 2311 LLT DstEltTy = DstTy.getElementType(); 2312 LLT DstCastTy = DstEltTy; // Intermediate bitcast result type 2313 LLT SrcPartTy = SrcEltTy; // Original unmerge result type. 2314 2315 // If there's an element size mismatch, insert intermediate casts to match 2316 // the result element type. 2317 if (NumSrcElt < NumDstElt) { // Source element type is larger. 2318 // %1:_(<4 x s8>) = G_BITCAST %0:_(<2 x s16>) 2319 // 2320 // => 2321 // 2322 // %2:_(s16), %3:_(s16) = G_UNMERGE_VALUES %0 2323 // %3:_(<2 x s8>) = G_BITCAST %2 2324 // %4:_(<2 x s8>) = G_BITCAST %3 2325 // %1:_(<4 x s16>) = G_CONCAT_VECTORS %3, %4 2326 DstCastTy = LLT::vector(NumDstElt / NumSrcElt, DstEltTy); 2327 SrcPartTy = SrcEltTy; 2328 } else if (NumSrcElt > NumDstElt) { // Source element type is smaller. 2329 // 2330 // %1:_(<2 x s16>) = G_BITCAST %0:_(<4 x s8>) 2331 // 2332 // => 2333 // 2334 // %2:_(<2 x s8>), %3:_(<2 x s8>) = G_UNMERGE_VALUES %0 2335 // %3:_(s16) = G_BITCAST %2 2336 // %4:_(s16) = G_BITCAST %3 2337 // %1:_(<2 x s16>) = G_BUILD_VECTOR %3, %4 2338 SrcPartTy = LLT::vector(NumSrcElt / NumDstElt, SrcEltTy); 2339 DstCastTy = DstEltTy; 2340 } 2341 2342 getUnmergePieces(SrcRegs, MIRBuilder, Src, SrcPartTy); 2343 for (Register &SrcReg : SrcRegs) 2344 SrcReg = MIRBuilder.buildBitcast(DstCastTy, SrcReg).getReg(0); 2345 } else 2346 getUnmergePieces(SrcRegs, MIRBuilder, Src, SrcEltTy); 2347 2348 MIRBuilder.buildMerge(Dst, SrcRegs); 2349 MI.eraseFromParent(); 2350 return Legalized; 2351 } 2352 2353 if (DstTy.isVector()) { 2354 SmallVector<Register, 8> SrcRegs; 2355 getUnmergePieces(SrcRegs, MIRBuilder, Src, DstTy.getElementType()); 2356 MIRBuilder.buildMerge(Dst, SrcRegs); 2357 MI.eraseFromParent(); 2358 return Legalized; 2359 } 2360 2361 return UnableToLegalize; 2362 } 2363 2364 /// Perform a G_EXTRACT_VECTOR_ELT in a different sized vector element. If this 2365 /// is casting to a vector with a smaller element size, perform multiple element 2366 /// extracts and merge the results. If this is coercing to a vector with larger 2367 /// elements, index the bitcasted vector and extract the target element with bit 2368 /// operations. This is intended to force the indexing in the native register 2369 /// size for architectures that can dynamically index the register file. 2370 LegalizerHelper::LegalizeResult 2371 LegalizerHelper::bitcastExtractVectorElt(MachineInstr &MI, unsigned TypeIdx, 2372 LLT CastTy) { 2373 if (TypeIdx != 1) 2374 return UnableToLegalize; 2375 2376 Register Dst = MI.getOperand(0).getReg(); 2377 Register SrcVec = MI.getOperand(1).getReg(); 2378 Register Idx = MI.getOperand(2).getReg(); 2379 LLT SrcVecTy = MRI.getType(SrcVec); 2380 LLT IdxTy = MRI.getType(Idx); 2381 2382 LLT SrcEltTy = SrcVecTy.getElementType(); 2383 unsigned NewNumElts = CastTy.isVector() ? CastTy.getNumElements() : 1; 2384 unsigned OldNumElts = SrcVecTy.getNumElements(); 2385 2386 LLT NewEltTy = CastTy.isVector() ? CastTy.getElementType() : CastTy; 2387 Register CastVec = MIRBuilder.buildBitcast(CastTy, SrcVec).getReg(0); 2388 2389 const unsigned NewEltSize = NewEltTy.getSizeInBits(); 2390 const unsigned OldEltSize = SrcEltTy.getSizeInBits(); 2391 if (NewNumElts > OldNumElts) { 2392 // Decreasing the vector element size 2393 // 2394 // e.g. i64 = extract_vector_elt x:v2i64, y:i32 2395 // => 2396 // v4i32:castx = bitcast x:v2i64 2397 // 2398 // i64 = bitcast 2399 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))), 2400 // (i32 (extract_vector_elt castx, (2 * y + 1))) 2401 // 2402 if (NewNumElts % OldNumElts != 0) 2403 return UnableToLegalize; 2404 2405 // Type of the intermediate result vector. 2406 const unsigned NewEltsPerOldElt = NewNumElts / OldNumElts; 2407 LLT MidTy = LLT::scalarOrVector(NewEltsPerOldElt, NewEltTy); 2408 2409 auto NewEltsPerOldEltK = MIRBuilder.buildConstant(IdxTy, NewEltsPerOldElt); 2410 2411 SmallVector<Register, 8> NewOps(NewEltsPerOldElt); 2412 auto NewBaseIdx = MIRBuilder.buildMul(IdxTy, Idx, NewEltsPerOldEltK); 2413 2414 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) { 2415 auto IdxOffset = MIRBuilder.buildConstant(IdxTy, I); 2416 auto TmpIdx = MIRBuilder.buildAdd(IdxTy, NewBaseIdx, IdxOffset); 2417 auto Elt = MIRBuilder.buildExtractVectorElement(NewEltTy, CastVec, TmpIdx); 2418 NewOps[I] = Elt.getReg(0); 2419 } 2420 2421 auto NewVec = MIRBuilder.buildBuildVector(MidTy, NewOps); 2422 MIRBuilder.buildBitcast(Dst, NewVec); 2423 MI.eraseFromParent(); 2424 return Legalized; 2425 } 2426 2427 if (NewNumElts < OldNumElts) { 2428 if (NewEltSize % OldEltSize != 0) 2429 return UnableToLegalize; 2430 2431 // This only depends on powers of 2 because we use bit tricks to figure out 2432 // the bit offset we need to shift to get the target element. A general 2433 // expansion could emit division/multiply. 2434 if (!isPowerOf2_32(NewEltSize / OldEltSize)) 2435 return UnableToLegalize; 2436 2437 // Increasing the vector element size. 2438 // %elt:_(small_elt) = G_EXTRACT_VECTOR_ELT %vec:_(<N x small_elt>), %idx 2439 // 2440 // => 2441 // 2442 // %cast = G_BITCAST %vec 2443 // %scaled_idx = G_LSHR %idx, Log2(DstEltSize / SrcEltSize) 2444 // %wide_elt = G_EXTRACT_VECTOR_ELT %cast, %scaled_idx 2445 // %offset_idx = G_AND %idx, ~(-1 << Log2(DstEltSize / SrcEltSize)) 2446 // %offset_bits = G_SHL %offset_idx, Log2(SrcEltSize) 2447 // %elt_bits = G_LSHR %wide_elt, %offset_bits 2448 // %elt = G_TRUNC %elt_bits 2449 2450 const unsigned Log2EltRatio = Log2_32(NewEltSize / OldEltSize); 2451 auto Log2Ratio = MIRBuilder.buildConstant(IdxTy, Log2EltRatio); 2452 2453 // Divide to get the index in the wider element type. 2454 auto ScaledIdx = MIRBuilder.buildLShr(IdxTy, Idx, Log2Ratio); 2455 2456 Register WideElt = CastVec; 2457 if (CastTy.isVector()) { 2458 WideElt = MIRBuilder.buildExtractVectorElement(NewEltTy, CastVec, 2459 ScaledIdx).getReg(0); 2460 } 2461 2462 // Now figure out the amount we need to shift to get the target bits. 2463 auto OffsetMask = MIRBuilder.buildConstant( 2464 IdxTy, ~(APInt::getAllOnesValue(IdxTy.getSizeInBits()) << Log2EltRatio)); 2465 auto OffsetIdx = MIRBuilder.buildAnd(IdxTy, Idx, OffsetMask); 2466 auto OffsetBits = MIRBuilder.buildShl( 2467 IdxTy, OffsetIdx, 2468 MIRBuilder.buildConstant(IdxTy, Log2_32(OldEltSize))); 2469 2470 // Shift the wide element to get the target element. 2471 auto ExtractedBits = MIRBuilder.buildLShr(NewEltTy, WideElt, OffsetBits); 2472 MIRBuilder.buildTrunc(Dst, ExtractedBits); 2473 MI.eraseFromParent(); 2474 return Legalized; 2475 } 2476 2477 return UnableToLegalize; 2478 } 2479 2480 LegalizerHelper::LegalizeResult 2481 LegalizerHelper::lowerLoad(MachineInstr &MI) { 2482 // Lower to a memory-width G_LOAD and a G_SEXT/G_ZEXT/G_ANYEXT 2483 Register DstReg = MI.getOperand(0).getReg(); 2484 Register PtrReg = MI.getOperand(1).getReg(); 2485 LLT DstTy = MRI.getType(DstReg); 2486 auto &MMO = **MI.memoperands_begin(); 2487 2488 if (DstTy.getSizeInBits() == MMO.getSizeInBits()) { 2489 if (MI.getOpcode() == TargetOpcode::G_LOAD) { 2490 // This load needs splitting into power of 2 sized loads. 2491 if (DstTy.isVector()) 2492 return UnableToLegalize; 2493 if (isPowerOf2_32(DstTy.getSizeInBits())) 2494 return UnableToLegalize; // Don't know what we're being asked to do. 2495 2496 // Our strategy here is to generate anyextending loads for the smaller 2497 // types up to next power-2 result type, and then combine the two larger 2498 // result values together, before truncating back down to the non-pow-2 2499 // type. 2500 // E.g. v1 = i24 load => 2501 // v2 = i32 zextload (2 byte) 2502 // v3 = i32 load (1 byte) 2503 // v4 = i32 shl v3, 16 2504 // v5 = i32 or v4, v2 2505 // v1 = i24 trunc v5 2506 // By doing this we generate the correct truncate which should get 2507 // combined away as an artifact with a matching extend. 2508 uint64_t LargeSplitSize = PowerOf2Floor(DstTy.getSizeInBits()); 2509 uint64_t SmallSplitSize = DstTy.getSizeInBits() - LargeSplitSize; 2510 2511 MachineFunction &MF = MIRBuilder.getMF(); 2512 MachineMemOperand *LargeMMO = 2513 MF.getMachineMemOperand(&MMO, 0, LargeSplitSize / 8); 2514 MachineMemOperand *SmallMMO = MF.getMachineMemOperand( 2515 &MMO, LargeSplitSize / 8, SmallSplitSize / 8); 2516 2517 LLT PtrTy = MRI.getType(PtrReg); 2518 unsigned AnyExtSize = NextPowerOf2(DstTy.getSizeInBits()); 2519 LLT AnyExtTy = LLT::scalar(AnyExtSize); 2520 Register LargeLdReg = MRI.createGenericVirtualRegister(AnyExtTy); 2521 Register SmallLdReg = MRI.createGenericVirtualRegister(AnyExtTy); 2522 auto LargeLoad = MIRBuilder.buildLoadInstr( 2523 TargetOpcode::G_ZEXTLOAD, LargeLdReg, PtrReg, *LargeMMO); 2524 2525 auto OffsetCst = MIRBuilder.buildConstant( 2526 LLT::scalar(PtrTy.getSizeInBits()), LargeSplitSize / 8); 2527 Register PtrAddReg = MRI.createGenericVirtualRegister(PtrTy); 2528 auto SmallPtr = 2529 MIRBuilder.buildPtrAdd(PtrAddReg, PtrReg, OffsetCst.getReg(0)); 2530 auto SmallLoad = MIRBuilder.buildLoad(SmallLdReg, SmallPtr.getReg(0), 2531 *SmallMMO); 2532 2533 auto ShiftAmt = MIRBuilder.buildConstant(AnyExtTy, LargeSplitSize); 2534 auto Shift = MIRBuilder.buildShl(AnyExtTy, SmallLoad, ShiftAmt); 2535 auto Or = MIRBuilder.buildOr(AnyExtTy, Shift, LargeLoad); 2536 MIRBuilder.buildTrunc(DstReg, {Or.getReg(0)}); 2537 MI.eraseFromParent(); 2538 return Legalized; 2539 } 2540 2541 MIRBuilder.buildLoad(DstReg, PtrReg, MMO); 2542 MI.eraseFromParent(); 2543 return Legalized; 2544 } 2545 2546 if (DstTy.isScalar()) { 2547 Register TmpReg = 2548 MRI.createGenericVirtualRegister(LLT::scalar(MMO.getSizeInBits())); 2549 MIRBuilder.buildLoad(TmpReg, PtrReg, MMO); 2550 switch (MI.getOpcode()) { 2551 default: 2552 llvm_unreachable("Unexpected opcode"); 2553 case TargetOpcode::G_LOAD: 2554 MIRBuilder.buildAnyExtOrTrunc(DstReg, TmpReg); 2555 break; 2556 case TargetOpcode::G_SEXTLOAD: 2557 MIRBuilder.buildSExt(DstReg, TmpReg); 2558 break; 2559 case TargetOpcode::G_ZEXTLOAD: 2560 MIRBuilder.buildZExt(DstReg, TmpReg); 2561 break; 2562 } 2563 2564 MI.eraseFromParent(); 2565 return Legalized; 2566 } 2567 2568 return UnableToLegalize; 2569 } 2570 2571 LegalizerHelper::LegalizeResult 2572 LegalizerHelper::lowerStore(MachineInstr &MI) { 2573 // Lower a non-power of 2 store into multiple pow-2 stores. 2574 // E.g. split an i24 store into an i16 store + i8 store. 2575 // We do this by first extending the stored value to the next largest power 2576 // of 2 type, and then using truncating stores to store the components. 2577 // By doing this, likewise with G_LOAD, generate an extend that can be 2578 // artifact-combined away instead of leaving behind extracts. 2579 Register SrcReg = MI.getOperand(0).getReg(); 2580 Register PtrReg = MI.getOperand(1).getReg(); 2581 LLT SrcTy = MRI.getType(SrcReg); 2582 MachineMemOperand &MMO = **MI.memoperands_begin(); 2583 if (SrcTy.getSizeInBits() != MMO.getSizeInBits()) 2584 return UnableToLegalize; 2585 if (SrcTy.isVector()) 2586 return UnableToLegalize; 2587 if (isPowerOf2_32(SrcTy.getSizeInBits())) 2588 return UnableToLegalize; // Don't know what we're being asked to do. 2589 2590 // Extend to the next pow-2. 2591 const LLT ExtendTy = LLT::scalar(NextPowerOf2(SrcTy.getSizeInBits())); 2592 auto ExtVal = MIRBuilder.buildAnyExt(ExtendTy, SrcReg); 2593 2594 // Obtain the smaller value by shifting away the larger value. 2595 uint64_t LargeSplitSize = PowerOf2Floor(SrcTy.getSizeInBits()); 2596 uint64_t SmallSplitSize = SrcTy.getSizeInBits() - LargeSplitSize; 2597 auto ShiftAmt = MIRBuilder.buildConstant(ExtendTy, LargeSplitSize); 2598 auto SmallVal = MIRBuilder.buildLShr(ExtendTy, ExtVal, ShiftAmt); 2599 2600 // Generate the PtrAdd and truncating stores. 2601 LLT PtrTy = MRI.getType(PtrReg); 2602 auto OffsetCst = MIRBuilder.buildConstant( 2603 LLT::scalar(PtrTy.getSizeInBits()), LargeSplitSize / 8); 2604 Register PtrAddReg = MRI.createGenericVirtualRegister(PtrTy); 2605 auto SmallPtr = 2606 MIRBuilder.buildPtrAdd(PtrAddReg, PtrReg, OffsetCst.getReg(0)); 2607 2608 MachineFunction &MF = MIRBuilder.getMF(); 2609 MachineMemOperand *LargeMMO = 2610 MF.getMachineMemOperand(&MMO, 0, LargeSplitSize / 8); 2611 MachineMemOperand *SmallMMO = 2612 MF.getMachineMemOperand(&MMO, LargeSplitSize / 8, SmallSplitSize / 8); 2613 MIRBuilder.buildStore(ExtVal.getReg(0), PtrReg, *LargeMMO); 2614 MIRBuilder.buildStore(SmallVal.getReg(0), SmallPtr.getReg(0), *SmallMMO); 2615 MI.eraseFromParent(); 2616 return Legalized; 2617 } 2618 2619 LegalizerHelper::LegalizeResult 2620 LegalizerHelper::bitcast(MachineInstr &MI, unsigned TypeIdx, LLT CastTy) { 2621 switch (MI.getOpcode()) { 2622 case TargetOpcode::G_LOAD: { 2623 if (TypeIdx != 0) 2624 return UnableToLegalize; 2625 2626 Observer.changingInstr(MI); 2627 bitcastDst(MI, CastTy, 0); 2628 Observer.changedInstr(MI); 2629 return Legalized; 2630 } 2631 case TargetOpcode::G_STORE: { 2632 if (TypeIdx != 0) 2633 return UnableToLegalize; 2634 2635 Observer.changingInstr(MI); 2636 bitcastSrc(MI, CastTy, 0); 2637 Observer.changedInstr(MI); 2638 return Legalized; 2639 } 2640 case TargetOpcode::G_SELECT: { 2641 if (TypeIdx != 0) 2642 return UnableToLegalize; 2643 2644 if (MRI.getType(MI.getOperand(1).getReg()).isVector()) { 2645 LLVM_DEBUG( 2646 dbgs() << "bitcast action not implemented for vector select\n"); 2647 return UnableToLegalize; 2648 } 2649 2650 Observer.changingInstr(MI); 2651 bitcastSrc(MI, CastTy, 2); 2652 bitcastSrc(MI, CastTy, 3); 2653 bitcastDst(MI, CastTy, 0); 2654 Observer.changedInstr(MI); 2655 return Legalized; 2656 } 2657 case TargetOpcode::G_AND: 2658 case TargetOpcode::G_OR: 2659 case TargetOpcode::G_XOR: { 2660 Observer.changingInstr(MI); 2661 bitcastSrc(MI, CastTy, 1); 2662 bitcastSrc(MI, CastTy, 2); 2663 bitcastDst(MI, CastTy, 0); 2664 Observer.changedInstr(MI); 2665 return Legalized; 2666 } 2667 case TargetOpcode::G_EXTRACT_VECTOR_ELT: 2668 return bitcastExtractVectorElt(MI, TypeIdx, CastTy); 2669 default: 2670 return UnableToLegalize; 2671 } 2672 } 2673 2674 // Legalize an instruction by changing the opcode in place. 2675 void LegalizerHelper::changeOpcode(MachineInstr &MI, unsigned NewOpcode) { 2676 Observer.changingInstr(MI); 2677 MI.setDesc(MIRBuilder.getTII().get(NewOpcode)); 2678 Observer.changedInstr(MI); 2679 } 2680 2681 LegalizerHelper::LegalizeResult 2682 LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { 2683 using namespace TargetOpcode; 2684 2685 switch(MI.getOpcode()) { 2686 default: 2687 return UnableToLegalize; 2688 case TargetOpcode::G_BITCAST: 2689 return lowerBitcast(MI); 2690 case TargetOpcode::G_SREM: 2691 case TargetOpcode::G_UREM: { 2692 auto Quot = 2693 MIRBuilder.buildInstr(MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV, {Ty}, 2694 {MI.getOperand(1), MI.getOperand(2)}); 2695 2696 auto Prod = MIRBuilder.buildMul(Ty, Quot, MI.getOperand(2)); 2697 MIRBuilder.buildSub(MI.getOperand(0), MI.getOperand(1), Prod); 2698 MI.eraseFromParent(); 2699 return Legalized; 2700 } 2701 case TargetOpcode::G_SADDO: 2702 case TargetOpcode::G_SSUBO: 2703 return lowerSADDO_SSUBO(MI); 2704 case TargetOpcode::G_SMULO: 2705 case TargetOpcode::G_UMULO: { 2706 // Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the 2707 // result. 2708 Register Res = MI.getOperand(0).getReg(); 2709 Register Overflow = MI.getOperand(1).getReg(); 2710 Register LHS = MI.getOperand(2).getReg(); 2711 Register RHS = MI.getOperand(3).getReg(); 2712 2713 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO 2714 ? TargetOpcode::G_SMULH 2715 : TargetOpcode::G_UMULH; 2716 2717 Observer.changingInstr(MI); 2718 const auto &TII = MIRBuilder.getTII(); 2719 MI.setDesc(TII.get(TargetOpcode::G_MUL)); 2720 MI.RemoveOperand(1); 2721 Observer.changedInstr(MI); 2722 2723 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); 2724 2725 auto HiPart = MIRBuilder.buildInstr(Opcode, {Ty}, {LHS, RHS}); 2726 auto Zero = MIRBuilder.buildConstant(Ty, 0); 2727 2728 // For *signed* multiply, overflow is detected by checking: 2729 // (hi != (lo >> bitwidth-1)) 2730 if (Opcode == TargetOpcode::G_SMULH) { 2731 auto ShiftAmt = MIRBuilder.buildConstant(Ty, Ty.getSizeInBits() - 1); 2732 auto Shifted = MIRBuilder.buildAShr(Ty, Res, ShiftAmt); 2733 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Shifted); 2734 } else { 2735 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Zero); 2736 } 2737 return Legalized; 2738 } 2739 case TargetOpcode::G_FNEG: { 2740 // TODO: Handle vector types once we are able to 2741 // represent them. 2742 if (Ty.isVector()) 2743 return UnableToLegalize; 2744 Register Res = MI.getOperand(0).getReg(); 2745 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext(); 2746 Type *ZeroTy = getFloatTypeForLLT(Ctx, Ty); 2747 if (!ZeroTy) 2748 return UnableToLegalize; 2749 ConstantFP &ZeroForNegation = 2750 *cast<ConstantFP>(ConstantFP::getZeroValueForNegation(ZeroTy)); 2751 auto Zero = MIRBuilder.buildFConstant(Ty, ZeroForNegation); 2752 Register SubByReg = MI.getOperand(1).getReg(); 2753 Register ZeroReg = Zero.getReg(0); 2754 MIRBuilder.buildFSub(Res, ZeroReg, SubByReg, MI.getFlags()); 2755 MI.eraseFromParent(); 2756 return Legalized; 2757 } 2758 case TargetOpcode::G_FSUB: { 2759 // Lower (G_FSUB LHS, RHS) to (G_FADD LHS, (G_FNEG RHS)). 2760 // First, check if G_FNEG is marked as Lower. If so, we may 2761 // end up with an infinite loop as G_FSUB is used to legalize G_FNEG. 2762 if (LI.getAction({G_FNEG, {Ty}}).Action == Lower) 2763 return UnableToLegalize; 2764 Register Res = MI.getOperand(0).getReg(); 2765 Register LHS = MI.getOperand(1).getReg(); 2766 Register RHS = MI.getOperand(2).getReg(); 2767 Register Neg = MRI.createGenericVirtualRegister(Ty); 2768 MIRBuilder.buildFNeg(Neg, RHS); 2769 MIRBuilder.buildFAdd(Res, LHS, Neg, MI.getFlags()); 2770 MI.eraseFromParent(); 2771 return Legalized; 2772 } 2773 case TargetOpcode::G_FMAD: 2774 return lowerFMad(MI); 2775 case TargetOpcode::G_FFLOOR: 2776 return lowerFFloor(MI); 2777 case TargetOpcode::G_INTRINSIC_ROUND: 2778 return lowerIntrinsicRound(MI); 2779 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: { 2780 // Since round even is the assumed rounding mode for unconstrained FP 2781 // operations, rint and roundeven are the same operation. 2782 changeOpcode(MI, TargetOpcode::G_FRINT); 2783 return Legalized; 2784 } 2785 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: { 2786 Register OldValRes = MI.getOperand(0).getReg(); 2787 Register SuccessRes = MI.getOperand(1).getReg(); 2788 Register Addr = MI.getOperand(2).getReg(); 2789 Register CmpVal = MI.getOperand(3).getReg(); 2790 Register NewVal = MI.getOperand(4).getReg(); 2791 MIRBuilder.buildAtomicCmpXchg(OldValRes, Addr, CmpVal, NewVal, 2792 **MI.memoperands_begin()); 2793 MIRBuilder.buildICmp(CmpInst::ICMP_EQ, SuccessRes, OldValRes, CmpVal); 2794 MI.eraseFromParent(); 2795 return Legalized; 2796 } 2797 case TargetOpcode::G_LOAD: 2798 case TargetOpcode::G_SEXTLOAD: 2799 case TargetOpcode::G_ZEXTLOAD: 2800 return lowerLoad(MI); 2801 case TargetOpcode::G_STORE: 2802 return lowerStore(MI); 2803 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 2804 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 2805 case TargetOpcode::G_CTLZ: 2806 case TargetOpcode::G_CTTZ: 2807 case TargetOpcode::G_CTPOP: 2808 return lowerBitCount(MI, TypeIdx, Ty); 2809 case G_UADDO: { 2810 Register Res = MI.getOperand(0).getReg(); 2811 Register CarryOut = MI.getOperand(1).getReg(); 2812 Register LHS = MI.getOperand(2).getReg(); 2813 Register RHS = MI.getOperand(3).getReg(); 2814 2815 MIRBuilder.buildAdd(Res, LHS, RHS); 2816 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, RHS); 2817 2818 MI.eraseFromParent(); 2819 return Legalized; 2820 } 2821 case G_UADDE: { 2822 Register Res = MI.getOperand(0).getReg(); 2823 Register CarryOut = MI.getOperand(1).getReg(); 2824 Register LHS = MI.getOperand(2).getReg(); 2825 Register RHS = MI.getOperand(3).getReg(); 2826 Register CarryIn = MI.getOperand(4).getReg(); 2827 LLT Ty = MRI.getType(Res); 2828 2829 auto TmpRes = MIRBuilder.buildAdd(Ty, LHS, RHS); 2830 auto ZExtCarryIn = MIRBuilder.buildZExt(Ty, CarryIn); 2831 MIRBuilder.buildAdd(Res, TmpRes, ZExtCarryIn); 2832 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, LHS); 2833 2834 MI.eraseFromParent(); 2835 return Legalized; 2836 } 2837 case G_USUBO: { 2838 Register Res = MI.getOperand(0).getReg(); 2839 Register BorrowOut = MI.getOperand(1).getReg(); 2840 Register LHS = MI.getOperand(2).getReg(); 2841 Register RHS = MI.getOperand(3).getReg(); 2842 2843 MIRBuilder.buildSub(Res, LHS, RHS); 2844 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, BorrowOut, LHS, RHS); 2845 2846 MI.eraseFromParent(); 2847 return Legalized; 2848 } 2849 case G_USUBE: { 2850 Register Res = MI.getOperand(0).getReg(); 2851 Register BorrowOut = MI.getOperand(1).getReg(); 2852 Register LHS = MI.getOperand(2).getReg(); 2853 Register RHS = MI.getOperand(3).getReg(); 2854 Register BorrowIn = MI.getOperand(4).getReg(); 2855 const LLT CondTy = MRI.getType(BorrowOut); 2856 const LLT Ty = MRI.getType(Res); 2857 2858 auto TmpRes = MIRBuilder.buildSub(Ty, LHS, RHS); 2859 auto ZExtBorrowIn = MIRBuilder.buildZExt(Ty, BorrowIn); 2860 MIRBuilder.buildSub(Res, TmpRes, ZExtBorrowIn); 2861 2862 auto LHS_EQ_RHS = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, CondTy, LHS, RHS); 2863 auto LHS_ULT_RHS = MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CondTy, LHS, RHS); 2864 MIRBuilder.buildSelect(BorrowOut, LHS_EQ_RHS, BorrowIn, LHS_ULT_RHS); 2865 2866 MI.eraseFromParent(); 2867 return Legalized; 2868 } 2869 case G_UITOFP: 2870 return lowerUITOFP(MI, TypeIdx, Ty); 2871 case G_SITOFP: 2872 return lowerSITOFP(MI, TypeIdx, Ty); 2873 case G_FPTOUI: 2874 return lowerFPTOUI(MI, TypeIdx, Ty); 2875 case G_FPTOSI: 2876 return lowerFPTOSI(MI); 2877 case G_FPTRUNC: 2878 return lowerFPTRUNC(MI, TypeIdx, Ty); 2879 case G_FPOWI: 2880 return lowerFPOWI(MI); 2881 case G_SMIN: 2882 case G_SMAX: 2883 case G_UMIN: 2884 case G_UMAX: 2885 return lowerMinMax(MI, TypeIdx, Ty); 2886 case G_FCOPYSIGN: 2887 return lowerFCopySign(MI, TypeIdx, Ty); 2888 case G_FMINNUM: 2889 case G_FMAXNUM: 2890 return lowerFMinNumMaxNum(MI); 2891 case G_MERGE_VALUES: 2892 return lowerMergeValues(MI); 2893 case G_UNMERGE_VALUES: 2894 return lowerUnmergeValues(MI); 2895 case TargetOpcode::G_SEXT_INREG: { 2896 assert(MI.getOperand(2).isImm() && "Expected immediate"); 2897 int64_t SizeInBits = MI.getOperand(2).getImm(); 2898 2899 Register DstReg = MI.getOperand(0).getReg(); 2900 Register SrcReg = MI.getOperand(1).getReg(); 2901 LLT DstTy = MRI.getType(DstReg); 2902 Register TmpRes = MRI.createGenericVirtualRegister(DstTy); 2903 2904 auto MIBSz = MIRBuilder.buildConstant(DstTy, DstTy.getScalarSizeInBits() - SizeInBits); 2905 MIRBuilder.buildShl(TmpRes, SrcReg, MIBSz->getOperand(0)); 2906 MIRBuilder.buildAShr(DstReg, TmpRes, MIBSz->getOperand(0)); 2907 MI.eraseFromParent(); 2908 return Legalized; 2909 } 2910 case G_EXTRACT_VECTOR_ELT: 2911 return lowerExtractVectorElt(MI); 2912 case G_SHUFFLE_VECTOR: 2913 return lowerShuffleVector(MI); 2914 case G_DYN_STACKALLOC: 2915 return lowerDynStackAlloc(MI); 2916 case G_EXTRACT: 2917 return lowerExtract(MI); 2918 case G_INSERT: 2919 return lowerInsert(MI); 2920 case G_BSWAP: 2921 return lowerBswap(MI); 2922 case G_BITREVERSE: 2923 return lowerBitreverse(MI); 2924 case G_READ_REGISTER: 2925 case G_WRITE_REGISTER: 2926 return lowerReadWriteRegister(MI); 2927 case G_UADDSAT: 2928 case G_USUBSAT: { 2929 // Try to make a reasonable guess about which lowering strategy to use. The 2930 // target can override this with custom lowering and calling the 2931 // implementation functions. 2932 LLT Ty = MRI.getType(MI.getOperand(0).getReg()); 2933 if (LI.isLegalOrCustom({G_UMIN, Ty})) 2934 return lowerAddSubSatToMinMax(MI); 2935 return lowerAddSubSatToAddoSubo(MI); 2936 } 2937 case G_SADDSAT: 2938 case G_SSUBSAT: { 2939 LLT Ty = MRI.getType(MI.getOperand(0).getReg()); 2940 2941 // FIXME: It would probably make more sense to see if G_SADDO is preferred, 2942 // since it's a shorter expansion. However, we would need to figure out the 2943 // preferred boolean type for the carry out for the query. 2944 if (LI.isLegalOrCustom({G_SMIN, Ty}) && LI.isLegalOrCustom({G_SMAX, Ty})) 2945 return lowerAddSubSatToMinMax(MI); 2946 return lowerAddSubSatToAddoSubo(MI); 2947 } 2948 } 2949 } 2950 2951 Align LegalizerHelper::getStackTemporaryAlignment(LLT Ty, 2952 Align MinAlign) const { 2953 // FIXME: We're missing a way to go back from LLT to llvm::Type to query the 2954 // datalayout for the preferred alignment. Also there should be a target hook 2955 // for this to allow targets to reduce the alignment and ignore the 2956 // datalayout. e.g. AMDGPU should always use a 4-byte alignment, regardless of 2957 // the type. 2958 return std::max(Align(PowerOf2Ceil(Ty.getSizeInBytes())), MinAlign); 2959 } 2960 2961 MachineInstrBuilder 2962 LegalizerHelper::createStackTemporary(TypeSize Bytes, Align Alignment, 2963 MachinePointerInfo &PtrInfo) { 2964 MachineFunction &MF = MIRBuilder.getMF(); 2965 const DataLayout &DL = MIRBuilder.getDataLayout(); 2966 int FrameIdx = MF.getFrameInfo().CreateStackObject(Bytes, Alignment, false); 2967 2968 unsigned AddrSpace = DL.getAllocaAddrSpace(); 2969 LLT FramePtrTy = LLT::pointer(AddrSpace, DL.getPointerSizeInBits(AddrSpace)); 2970 2971 PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIdx); 2972 return MIRBuilder.buildFrameIndex(FramePtrTy, FrameIdx); 2973 } 2974 2975 static Register clampDynamicVectorIndex(MachineIRBuilder &B, Register IdxReg, 2976 LLT VecTy) { 2977 int64_t IdxVal; 2978 if (mi_match(IdxReg, *B.getMRI(), m_ICst(IdxVal))) 2979 return IdxReg; 2980 2981 LLT IdxTy = B.getMRI()->getType(IdxReg); 2982 unsigned NElts = VecTy.getNumElements(); 2983 if (isPowerOf2_32(NElts)) { 2984 APInt Imm = APInt::getLowBitsSet(IdxTy.getSizeInBits(), Log2_32(NElts)); 2985 return B.buildAnd(IdxTy, IdxReg, B.buildConstant(IdxTy, Imm)).getReg(0); 2986 } 2987 2988 return B.buildUMin(IdxTy, IdxReg, B.buildConstant(IdxTy, NElts - 1)) 2989 .getReg(0); 2990 } 2991 2992 Register LegalizerHelper::getVectorElementPointer(Register VecPtr, LLT VecTy, 2993 Register Index) { 2994 LLT EltTy = VecTy.getElementType(); 2995 2996 // Calculate the element offset and add it to the pointer. 2997 unsigned EltSize = EltTy.getSizeInBits() / 8; // FIXME: should be ABI size. 2998 assert(EltSize * 8 == EltTy.getSizeInBits() && 2999 "Converting bits to bytes lost precision"); 3000 3001 Index = clampDynamicVectorIndex(MIRBuilder, Index, VecTy); 3002 3003 LLT IdxTy = MRI.getType(Index); 3004 auto Mul = MIRBuilder.buildMul(IdxTy, Index, 3005 MIRBuilder.buildConstant(IdxTy, EltSize)); 3006 3007 LLT PtrTy = MRI.getType(VecPtr); 3008 return MIRBuilder.buildPtrAdd(PtrTy, VecPtr, Mul).getReg(0); 3009 } 3010 3011 LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorImplicitDef( 3012 MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) { 3013 Register DstReg = MI.getOperand(0).getReg(); 3014 LLT DstTy = MRI.getType(DstReg); 3015 LLT LCMTy = getLCMType(DstTy, NarrowTy); 3016 3017 unsigned NumParts = LCMTy.getSizeInBits() / NarrowTy.getSizeInBits(); 3018 3019 auto NewUndef = MIRBuilder.buildUndef(NarrowTy); 3020 SmallVector<Register, 8> Parts(NumParts, NewUndef.getReg(0)); 3021 3022 buildWidenedRemergeToDst(DstReg, LCMTy, Parts); 3023 MI.eraseFromParent(); 3024 return Legalized; 3025 } 3026 3027 // Handle splitting vector operations which need to have the same number of 3028 // elements in each type index, but each type index may have a different element 3029 // type. 3030 // 3031 // e.g. <4 x s64> = G_SHL <4 x s64>, <4 x s32> -> 3032 // <2 x s64> = G_SHL <2 x s64>, <2 x s32> 3033 // <2 x s64> = G_SHL <2 x s64>, <2 x s32> 3034 // 3035 // Also handles some irregular breakdown cases, e.g. 3036 // e.g. <3 x s64> = G_SHL <3 x s64>, <3 x s32> -> 3037 // <2 x s64> = G_SHL <2 x s64>, <2 x s32> 3038 // s64 = G_SHL s64, s32 3039 LegalizerHelper::LegalizeResult 3040 LegalizerHelper::fewerElementsVectorMultiEltType( 3041 MachineInstr &MI, unsigned TypeIdx, LLT NarrowTyArg) { 3042 if (TypeIdx != 0) 3043 return UnableToLegalize; 3044 3045 const LLT NarrowTy0 = NarrowTyArg; 3046 const unsigned NewNumElts = 3047 NarrowTy0.isVector() ? NarrowTy0.getNumElements() : 1; 3048 3049 const Register DstReg = MI.getOperand(0).getReg(); 3050 LLT DstTy = MRI.getType(DstReg); 3051 LLT LeftoverTy0; 3052 3053 // All of the operands need to have the same number of elements, so if we can 3054 // determine a type breakdown for the result type, we can for all of the 3055 // source types. 3056 int NumParts = getNarrowTypeBreakDown(DstTy, NarrowTy0, LeftoverTy0).first; 3057 if (NumParts < 0) 3058 return UnableToLegalize; 3059 3060 SmallVector<MachineInstrBuilder, 4> NewInsts; 3061 3062 SmallVector<Register, 4> DstRegs, LeftoverDstRegs; 3063 SmallVector<Register, 4> PartRegs, LeftoverRegs; 3064 3065 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) { 3066 Register SrcReg = MI.getOperand(I).getReg(); 3067 LLT SrcTyI = MRI.getType(SrcReg); 3068 LLT NarrowTyI = LLT::scalarOrVector(NewNumElts, SrcTyI.getScalarType()); 3069 LLT LeftoverTyI; 3070 3071 // Split this operand into the requested typed registers, and any leftover 3072 // required to reproduce the original type. 3073 if (!extractParts(SrcReg, SrcTyI, NarrowTyI, LeftoverTyI, PartRegs, 3074 LeftoverRegs)) 3075 return UnableToLegalize; 3076 3077 if (I == 1) { 3078 // For the first operand, create an instruction for each part and setup 3079 // the result. 3080 for (Register PartReg : PartRegs) { 3081 Register PartDstReg = MRI.createGenericVirtualRegister(NarrowTy0); 3082 NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode()) 3083 .addDef(PartDstReg) 3084 .addUse(PartReg)); 3085 DstRegs.push_back(PartDstReg); 3086 } 3087 3088 for (Register LeftoverReg : LeftoverRegs) { 3089 Register PartDstReg = MRI.createGenericVirtualRegister(LeftoverTy0); 3090 NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode()) 3091 .addDef(PartDstReg) 3092 .addUse(LeftoverReg)); 3093 LeftoverDstRegs.push_back(PartDstReg); 3094 } 3095 } else { 3096 assert(NewInsts.size() == PartRegs.size() + LeftoverRegs.size()); 3097 3098 // Add the newly created operand splits to the existing instructions. The 3099 // odd-sized pieces are ordered after the requested NarrowTyArg sized 3100 // pieces. 3101 unsigned InstCount = 0; 3102 for (unsigned J = 0, JE = PartRegs.size(); J != JE; ++J) 3103 NewInsts[InstCount++].addUse(PartRegs[J]); 3104 for (unsigned J = 0, JE = LeftoverRegs.size(); J != JE; ++J) 3105 NewInsts[InstCount++].addUse(LeftoverRegs[J]); 3106 } 3107 3108 PartRegs.clear(); 3109 LeftoverRegs.clear(); 3110 } 3111 3112 // Insert the newly built operations and rebuild the result register. 3113 for (auto &MIB : NewInsts) 3114 MIRBuilder.insertInstr(MIB); 3115 3116 insertParts(DstReg, DstTy, NarrowTy0, DstRegs, LeftoverTy0, LeftoverDstRegs); 3117 3118 MI.eraseFromParent(); 3119 return Legalized; 3120 } 3121 3122 LegalizerHelper::LegalizeResult 3123 LegalizerHelper::fewerElementsVectorCasts(MachineInstr &MI, unsigned TypeIdx, 3124 LLT NarrowTy) { 3125 if (TypeIdx != 0) 3126 return UnableToLegalize; 3127 3128 Register DstReg = MI.getOperand(0).getReg(); 3129 Register SrcReg = MI.getOperand(1).getReg(); 3130 LLT DstTy = MRI.getType(DstReg); 3131 LLT SrcTy = MRI.getType(SrcReg); 3132 3133 LLT NarrowTy0 = NarrowTy; 3134 LLT NarrowTy1; 3135 unsigned NumParts; 3136 3137 if (NarrowTy.isVector()) { 3138 // Uneven breakdown not handled. 3139 NumParts = DstTy.getNumElements() / NarrowTy.getNumElements(); 3140 if (NumParts * NarrowTy.getNumElements() != DstTy.getNumElements()) 3141 return UnableToLegalize; 3142 3143 NarrowTy1 = LLT::vector(NumParts, SrcTy.getElementType().getSizeInBits()); 3144 } else { 3145 NumParts = DstTy.getNumElements(); 3146 NarrowTy1 = SrcTy.getElementType(); 3147 } 3148 3149 SmallVector<Register, 4> SrcRegs, DstRegs; 3150 extractParts(SrcReg, NarrowTy1, NumParts, SrcRegs); 3151 3152 for (unsigned I = 0; I < NumParts; ++I) { 3153 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0); 3154 MachineInstr *NewInst = 3155 MIRBuilder.buildInstr(MI.getOpcode(), {DstReg}, {SrcRegs[I]}); 3156 3157 NewInst->setFlags(MI.getFlags()); 3158 DstRegs.push_back(DstReg); 3159 } 3160 3161 if (NarrowTy.isVector()) 3162 MIRBuilder.buildConcatVectors(DstReg, DstRegs); 3163 else 3164 MIRBuilder.buildBuildVector(DstReg, DstRegs); 3165 3166 MI.eraseFromParent(); 3167 return Legalized; 3168 } 3169 3170 LegalizerHelper::LegalizeResult 3171 LegalizerHelper::fewerElementsVectorCmp(MachineInstr &MI, unsigned TypeIdx, 3172 LLT NarrowTy) { 3173 Register DstReg = MI.getOperand(0).getReg(); 3174 Register Src0Reg = MI.getOperand(2).getReg(); 3175 LLT DstTy = MRI.getType(DstReg); 3176 LLT SrcTy = MRI.getType(Src0Reg); 3177 3178 unsigned NumParts; 3179 LLT NarrowTy0, NarrowTy1; 3180 3181 if (TypeIdx == 0) { 3182 unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1; 3183 unsigned OldElts = DstTy.getNumElements(); 3184 3185 NarrowTy0 = NarrowTy; 3186 NumParts = NarrowTy.isVector() ? (OldElts / NewElts) : DstTy.getNumElements(); 3187 NarrowTy1 = NarrowTy.isVector() ? 3188 LLT::vector(NarrowTy.getNumElements(), SrcTy.getScalarSizeInBits()) : 3189 SrcTy.getElementType(); 3190 3191 } else { 3192 unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1; 3193 unsigned OldElts = SrcTy.getNumElements(); 3194 3195 NumParts = NarrowTy.isVector() ? (OldElts / NewElts) : 3196 NarrowTy.getNumElements(); 3197 NarrowTy0 = LLT::vector(NarrowTy.getNumElements(), 3198 DstTy.getScalarSizeInBits()); 3199 NarrowTy1 = NarrowTy; 3200 } 3201 3202 // FIXME: Don't know how to handle the situation where the small vectors 3203 // aren't all the same size yet. 3204 if (NarrowTy1.isVector() && 3205 NarrowTy1.getNumElements() * NumParts != DstTy.getNumElements()) 3206 return UnableToLegalize; 3207 3208 CmpInst::Predicate Pred 3209 = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate()); 3210 3211 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs; 3212 extractParts(MI.getOperand(2).getReg(), NarrowTy1, NumParts, Src1Regs); 3213 extractParts(MI.getOperand(3).getReg(), NarrowTy1, NumParts, Src2Regs); 3214 3215 for (unsigned I = 0; I < NumParts; ++I) { 3216 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0); 3217 DstRegs.push_back(DstReg); 3218 3219 if (MI.getOpcode() == TargetOpcode::G_ICMP) 3220 MIRBuilder.buildICmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]); 3221 else { 3222 MachineInstr *NewCmp 3223 = MIRBuilder.buildFCmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]); 3224 NewCmp->setFlags(MI.getFlags()); 3225 } 3226 } 3227 3228 if (NarrowTy1.isVector()) 3229 MIRBuilder.buildConcatVectors(DstReg, DstRegs); 3230 else 3231 MIRBuilder.buildBuildVector(DstReg, DstRegs); 3232 3233 MI.eraseFromParent(); 3234 return Legalized; 3235 } 3236 3237 LegalizerHelper::LegalizeResult 3238 LegalizerHelper::fewerElementsVectorSelect(MachineInstr &MI, unsigned TypeIdx, 3239 LLT NarrowTy) { 3240 Register DstReg = MI.getOperand(0).getReg(); 3241 Register CondReg = MI.getOperand(1).getReg(); 3242 3243 unsigned NumParts = 0; 3244 LLT NarrowTy0, NarrowTy1; 3245 3246 LLT DstTy = MRI.getType(DstReg); 3247 LLT CondTy = MRI.getType(CondReg); 3248 unsigned Size = DstTy.getSizeInBits(); 3249 3250 assert(TypeIdx == 0 || CondTy.isVector()); 3251 3252 if (TypeIdx == 0) { 3253 NarrowTy0 = NarrowTy; 3254 NarrowTy1 = CondTy; 3255 3256 unsigned NarrowSize = NarrowTy0.getSizeInBits(); 3257 // FIXME: Don't know how to handle the situation where the small vectors 3258 // aren't all the same size yet. 3259 if (Size % NarrowSize != 0) 3260 return UnableToLegalize; 3261 3262 NumParts = Size / NarrowSize; 3263 3264 // Need to break down the condition type 3265 if (CondTy.isVector()) { 3266 if (CondTy.getNumElements() == NumParts) 3267 NarrowTy1 = CondTy.getElementType(); 3268 else 3269 NarrowTy1 = LLT::vector(CondTy.getNumElements() / NumParts, 3270 CondTy.getScalarSizeInBits()); 3271 } 3272 } else { 3273 NumParts = CondTy.getNumElements(); 3274 if (NarrowTy.isVector()) { 3275 // TODO: Handle uneven breakdown. 3276 if (NumParts * NarrowTy.getNumElements() != CondTy.getNumElements()) 3277 return UnableToLegalize; 3278 3279 return UnableToLegalize; 3280 } else { 3281 NarrowTy0 = DstTy.getElementType(); 3282 NarrowTy1 = NarrowTy; 3283 } 3284 } 3285 3286 SmallVector<Register, 2> DstRegs, Src0Regs, Src1Regs, Src2Regs; 3287 if (CondTy.isVector()) 3288 extractParts(MI.getOperand(1).getReg(), NarrowTy1, NumParts, Src0Regs); 3289 3290 extractParts(MI.getOperand(2).getReg(), NarrowTy0, NumParts, Src1Regs); 3291 extractParts(MI.getOperand(3).getReg(), NarrowTy0, NumParts, Src2Regs); 3292 3293 for (unsigned i = 0; i < NumParts; ++i) { 3294 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0); 3295 MIRBuilder.buildSelect(DstReg, CondTy.isVector() ? Src0Regs[i] : CondReg, 3296 Src1Regs[i], Src2Regs[i]); 3297 DstRegs.push_back(DstReg); 3298 } 3299 3300 if (NarrowTy0.isVector()) 3301 MIRBuilder.buildConcatVectors(DstReg, DstRegs); 3302 else 3303 MIRBuilder.buildBuildVector(DstReg, DstRegs); 3304 3305 MI.eraseFromParent(); 3306 return Legalized; 3307 } 3308 3309 LegalizerHelper::LegalizeResult 3310 LegalizerHelper::fewerElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx, 3311 LLT NarrowTy) { 3312 const Register DstReg = MI.getOperand(0).getReg(); 3313 LLT PhiTy = MRI.getType(DstReg); 3314 LLT LeftoverTy; 3315 3316 // All of the operands need to have the same number of elements, so if we can 3317 // determine a type breakdown for the result type, we can for all of the 3318 // source types. 3319 int NumParts, NumLeftover; 3320 std::tie(NumParts, NumLeftover) 3321 = getNarrowTypeBreakDown(PhiTy, NarrowTy, LeftoverTy); 3322 if (NumParts < 0) 3323 return UnableToLegalize; 3324 3325 SmallVector<Register, 4> DstRegs, LeftoverDstRegs; 3326 SmallVector<MachineInstrBuilder, 4> NewInsts; 3327 3328 const int TotalNumParts = NumParts + NumLeftover; 3329 3330 // Insert the new phis in the result block first. 3331 for (int I = 0; I != TotalNumParts; ++I) { 3332 LLT Ty = I < NumParts ? NarrowTy : LeftoverTy; 3333 Register PartDstReg = MRI.createGenericVirtualRegister(Ty); 3334 NewInsts.push_back(MIRBuilder.buildInstr(TargetOpcode::G_PHI) 3335 .addDef(PartDstReg)); 3336 if (I < NumParts) 3337 DstRegs.push_back(PartDstReg); 3338 else 3339 LeftoverDstRegs.push_back(PartDstReg); 3340 } 3341 3342 MachineBasicBlock *MBB = MI.getParent(); 3343 MIRBuilder.setInsertPt(*MBB, MBB->getFirstNonPHI()); 3344 insertParts(DstReg, PhiTy, NarrowTy, DstRegs, LeftoverTy, LeftoverDstRegs); 3345 3346 SmallVector<Register, 4> PartRegs, LeftoverRegs; 3347 3348 // Insert code to extract the incoming values in each predecessor block. 3349 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) { 3350 PartRegs.clear(); 3351 LeftoverRegs.clear(); 3352 3353 Register SrcReg = MI.getOperand(I).getReg(); 3354 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB(); 3355 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator()); 3356 3357 LLT Unused; 3358 if (!extractParts(SrcReg, PhiTy, NarrowTy, Unused, PartRegs, 3359 LeftoverRegs)) 3360 return UnableToLegalize; 3361 3362 // Add the newly created operand splits to the existing instructions. The 3363 // odd-sized pieces are ordered after the requested NarrowTyArg sized 3364 // pieces. 3365 for (int J = 0; J != TotalNumParts; ++J) { 3366 MachineInstrBuilder MIB = NewInsts[J]; 3367 MIB.addUse(J < NumParts ? PartRegs[J] : LeftoverRegs[J - NumParts]); 3368 MIB.addMBB(&OpMBB); 3369 } 3370 } 3371 3372 MI.eraseFromParent(); 3373 return Legalized; 3374 } 3375 3376 LegalizerHelper::LegalizeResult 3377 LegalizerHelper::fewerElementsVectorUnmergeValues(MachineInstr &MI, 3378 unsigned TypeIdx, 3379 LLT NarrowTy) { 3380 if (TypeIdx != 1) 3381 return UnableToLegalize; 3382 3383 const int NumDst = MI.getNumOperands() - 1; 3384 const Register SrcReg = MI.getOperand(NumDst).getReg(); 3385 LLT SrcTy = MRI.getType(SrcReg); 3386 3387 LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); 3388 3389 // TODO: Create sequence of extracts. 3390 if (DstTy == NarrowTy) 3391 return UnableToLegalize; 3392 3393 LLT GCDTy = getGCDType(SrcTy, NarrowTy); 3394 if (DstTy == GCDTy) { 3395 // This would just be a copy of the same unmerge. 3396 // TODO: Create extracts, pad with undef and create intermediate merges. 3397 return UnableToLegalize; 3398 } 3399 3400 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg); 3401 const int NumUnmerge = Unmerge->getNumOperands() - 1; 3402 const int PartsPerUnmerge = NumDst / NumUnmerge; 3403 3404 for (int I = 0; I != NumUnmerge; ++I) { 3405 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES); 3406 3407 for (int J = 0; J != PartsPerUnmerge; ++J) 3408 MIB.addDef(MI.getOperand(I * PartsPerUnmerge + J).getReg()); 3409 MIB.addUse(Unmerge.getReg(I)); 3410 } 3411 3412 MI.eraseFromParent(); 3413 return Legalized; 3414 } 3415 3416 LegalizerHelper::LegalizeResult 3417 LegalizerHelper::fewerElementsVectorBuildVector(MachineInstr &MI, 3418 unsigned TypeIdx, 3419 LLT NarrowTy) { 3420 assert(TypeIdx == 0 && "not a vector type index"); 3421 Register DstReg = MI.getOperand(0).getReg(); 3422 LLT DstTy = MRI.getType(DstReg); 3423 LLT SrcTy = DstTy.getElementType(); 3424 3425 int DstNumElts = DstTy.getNumElements(); 3426 int NarrowNumElts = NarrowTy.getNumElements(); 3427 int NumConcat = (DstNumElts + NarrowNumElts - 1) / NarrowNumElts; 3428 LLT WidenedDstTy = LLT::vector(NarrowNumElts * NumConcat, SrcTy); 3429 3430 SmallVector<Register, 8> ConcatOps; 3431 SmallVector<Register, 8> SubBuildVector; 3432 3433 Register UndefReg; 3434 if (WidenedDstTy != DstTy) 3435 UndefReg = MIRBuilder.buildUndef(SrcTy).getReg(0); 3436 3437 // Create a G_CONCAT_VECTORS of NarrowTy pieces, padding with undef as 3438 // necessary. 3439 // 3440 // %3:_(<3 x s16>) = G_BUILD_VECTOR %0, %1, %2 3441 // -> <2 x s16> 3442 // 3443 // %4:_(s16) = G_IMPLICIT_DEF 3444 // %5:_(<2 x s16>) = G_BUILD_VECTOR %0, %1 3445 // %6:_(<2 x s16>) = G_BUILD_VECTOR %2, %4 3446 // %7:_(<4 x s16>) = G_CONCAT_VECTORS %5, %6 3447 // %3:_(<3 x s16>) = G_EXTRACT %7, 0 3448 for (int I = 0; I != NumConcat; ++I) { 3449 for (int J = 0; J != NarrowNumElts; ++J) { 3450 int SrcIdx = NarrowNumElts * I + J; 3451 3452 if (SrcIdx < DstNumElts) { 3453 Register SrcReg = MI.getOperand(SrcIdx + 1).getReg(); 3454 SubBuildVector.push_back(SrcReg); 3455 } else 3456 SubBuildVector.push_back(UndefReg); 3457 } 3458 3459 auto BuildVec = MIRBuilder.buildBuildVector(NarrowTy, SubBuildVector); 3460 ConcatOps.push_back(BuildVec.getReg(0)); 3461 SubBuildVector.clear(); 3462 } 3463 3464 if (DstTy == WidenedDstTy) 3465 MIRBuilder.buildConcatVectors(DstReg, ConcatOps); 3466 else { 3467 auto Concat = MIRBuilder.buildConcatVectors(WidenedDstTy, ConcatOps); 3468 MIRBuilder.buildExtract(DstReg, Concat, 0); 3469 } 3470 3471 MI.eraseFromParent(); 3472 return Legalized; 3473 } 3474 3475 LegalizerHelper::LegalizeResult 3476 LegalizerHelper::reduceLoadStoreWidth(MachineInstr &MI, unsigned TypeIdx, 3477 LLT NarrowTy) { 3478 // FIXME: Don't know how to handle secondary types yet. 3479 if (TypeIdx != 0) 3480 return UnableToLegalize; 3481 3482 MachineMemOperand *MMO = *MI.memoperands_begin(); 3483 3484 // This implementation doesn't work for atomics. Give up instead of doing 3485 // something invalid. 3486 if (MMO->getOrdering() != AtomicOrdering::NotAtomic || 3487 MMO->getFailureOrdering() != AtomicOrdering::NotAtomic) 3488 return UnableToLegalize; 3489 3490 bool IsLoad = MI.getOpcode() == TargetOpcode::G_LOAD; 3491 Register ValReg = MI.getOperand(0).getReg(); 3492 Register AddrReg = MI.getOperand(1).getReg(); 3493 LLT ValTy = MRI.getType(ValReg); 3494 3495 // FIXME: Do we need a distinct NarrowMemory legalize action? 3496 if (ValTy.getSizeInBits() != 8 * MMO->getSize()) { 3497 LLVM_DEBUG(dbgs() << "Can't narrow extload/truncstore\n"); 3498 return UnableToLegalize; 3499 } 3500 3501 int NumParts = -1; 3502 int NumLeftover = -1; 3503 LLT LeftoverTy; 3504 SmallVector<Register, 8> NarrowRegs, NarrowLeftoverRegs; 3505 if (IsLoad) { 3506 std::tie(NumParts, NumLeftover) = getNarrowTypeBreakDown(ValTy, NarrowTy, LeftoverTy); 3507 } else { 3508 if (extractParts(ValReg, ValTy, NarrowTy, LeftoverTy, NarrowRegs, 3509 NarrowLeftoverRegs)) { 3510 NumParts = NarrowRegs.size(); 3511 NumLeftover = NarrowLeftoverRegs.size(); 3512 } 3513 } 3514 3515 if (NumParts == -1) 3516 return UnableToLegalize; 3517 3518 LLT PtrTy = MRI.getType(AddrReg); 3519 const LLT OffsetTy = LLT::scalar(PtrTy.getSizeInBits()); 3520 3521 unsigned TotalSize = ValTy.getSizeInBits(); 3522 3523 // Split the load/store into PartTy sized pieces starting at Offset. If this 3524 // is a load, return the new registers in ValRegs. For a store, each elements 3525 // of ValRegs should be PartTy. Returns the next offset that needs to be 3526 // handled. 3527 auto splitTypePieces = [=](LLT PartTy, SmallVectorImpl<Register> &ValRegs, 3528 unsigned Offset) -> unsigned { 3529 MachineFunction &MF = MIRBuilder.getMF(); 3530 unsigned PartSize = PartTy.getSizeInBits(); 3531 for (unsigned Idx = 0, E = NumParts; Idx != E && Offset < TotalSize; 3532 Offset += PartSize, ++Idx) { 3533 unsigned ByteSize = PartSize / 8; 3534 unsigned ByteOffset = Offset / 8; 3535 Register NewAddrReg; 3536 3537 MIRBuilder.materializePtrAdd(NewAddrReg, AddrReg, OffsetTy, ByteOffset); 3538 3539 MachineMemOperand *NewMMO = 3540 MF.getMachineMemOperand(MMO, ByteOffset, ByteSize); 3541 3542 if (IsLoad) { 3543 Register Dst = MRI.createGenericVirtualRegister(PartTy); 3544 ValRegs.push_back(Dst); 3545 MIRBuilder.buildLoad(Dst, NewAddrReg, *NewMMO); 3546 } else { 3547 MIRBuilder.buildStore(ValRegs[Idx], NewAddrReg, *NewMMO); 3548 } 3549 } 3550 3551 return Offset; 3552 }; 3553 3554 unsigned HandledOffset = splitTypePieces(NarrowTy, NarrowRegs, 0); 3555 3556 // Handle the rest of the register if this isn't an even type breakdown. 3557 if (LeftoverTy.isValid()) 3558 splitTypePieces(LeftoverTy, NarrowLeftoverRegs, HandledOffset); 3559 3560 if (IsLoad) { 3561 insertParts(ValReg, ValTy, NarrowTy, NarrowRegs, 3562 LeftoverTy, NarrowLeftoverRegs); 3563 } 3564 3565 MI.eraseFromParent(); 3566 return Legalized; 3567 } 3568 3569 LegalizerHelper::LegalizeResult 3570 LegalizerHelper::reduceOperationWidth(MachineInstr &MI, unsigned int TypeIdx, 3571 LLT NarrowTy) { 3572 assert(TypeIdx == 0 && "only one type index expected"); 3573 3574 const unsigned Opc = MI.getOpcode(); 3575 const int NumOps = MI.getNumOperands() - 1; 3576 const Register DstReg = MI.getOperand(0).getReg(); 3577 const unsigned Flags = MI.getFlags(); 3578 const unsigned NarrowSize = NarrowTy.getSizeInBits(); 3579 const LLT NarrowScalarTy = LLT::scalar(NarrowSize); 3580 3581 assert(NumOps <= 3 && "expected instruction with 1 result and 1-3 sources"); 3582 3583 // First of all check whether we are narrowing (changing the element type) 3584 // or reducing the vector elements 3585 const LLT DstTy = MRI.getType(DstReg); 3586 const bool IsNarrow = NarrowTy.getScalarType() != DstTy.getScalarType(); 3587 3588 SmallVector<Register, 8> ExtractedRegs[3]; 3589 SmallVector<Register, 8> Parts; 3590 3591 unsigned NarrowElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1; 3592 3593 // Break down all the sources into NarrowTy pieces we can operate on. This may 3594 // involve creating merges to a wider type, padded with undef. 3595 for (int I = 0; I != NumOps; ++I) { 3596 Register SrcReg = MI.getOperand(I + 1).getReg(); 3597 LLT SrcTy = MRI.getType(SrcReg); 3598 3599 // The type to narrow SrcReg to. For narrowing, this is a smaller scalar. 3600 // For fewerElements, this is a smaller vector with the same element type. 3601 LLT OpNarrowTy; 3602 if (IsNarrow) { 3603 OpNarrowTy = NarrowScalarTy; 3604 3605 // In case of narrowing, we need to cast vectors to scalars for this to 3606 // work properly 3607 // FIXME: Can we do without the bitcast here if we're narrowing? 3608 if (SrcTy.isVector()) { 3609 SrcTy = LLT::scalar(SrcTy.getSizeInBits()); 3610 SrcReg = MIRBuilder.buildBitcast(SrcTy, SrcReg).getReg(0); 3611 } 3612 } else { 3613 OpNarrowTy = LLT::scalarOrVector(NarrowElts, SrcTy.getScalarType()); 3614 } 3615 3616 LLT GCDTy = extractGCDType(ExtractedRegs[I], SrcTy, OpNarrowTy, SrcReg); 3617 3618 // Build a sequence of NarrowTy pieces in ExtractedRegs for this operand. 3619 buildLCMMergePieces(SrcTy, OpNarrowTy, GCDTy, ExtractedRegs[I], 3620 TargetOpcode::G_ANYEXT); 3621 } 3622 3623 SmallVector<Register, 8> ResultRegs; 3624 3625 // Input operands for each sub-instruction. 3626 SmallVector<SrcOp, 4> InputRegs(NumOps, Register()); 3627 3628 int NumParts = ExtractedRegs[0].size(); 3629 const unsigned DstSize = DstTy.getSizeInBits(); 3630 const LLT DstScalarTy = LLT::scalar(DstSize); 3631 3632 // Narrowing needs to use scalar types 3633 LLT DstLCMTy, NarrowDstTy; 3634 if (IsNarrow) { 3635 DstLCMTy = getLCMType(DstScalarTy, NarrowScalarTy); 3636 NarrowDstTy = NarrowScalarTy; 3637 } else { 3638 DstLCMTy = getLCMType(DstTy, NarrowTy); 3639 NarrowDstTy = NarrowTy; 3640 } 3641 3642 // We widened the source registers to satisfy merge/unmerge size 3643 // constraints. We'll have some extra fully undef parts. 3644 const int NumRealParts = (DstSize + NarrowSize - 1) / NarrowSize; 3645 3646 for (int I = 0; I != NumRealParts; ++I) { 3647 // Emit this instruction on each of the split pieces. 3648 for (int J = 0; J != NumOps; ++J) 3649 InputRegs[J] = ExtractedRegs[J][I]; 3650 3651 auto Inst = MIRBuilder.buildInstr(Opc, {NarrowDstTy}, InputRegs, Flags); 3652 ResultRegs.push_back(Inst.getReg(0)); 3653 } 3654 3655 // Fill out the widened result with undef instead of creating instructions 3656 // with undef inputs. 3657 int NumUndefParts = NumParts - NumRealParts; 3658 if (NumUndefParts != 0) 3659 ResultRegs.append(NumUndefParts, 3660 MIRBuilder.buildUndef(NarrowDstTy).getReg(0)); 3661 3662 // Extract the possibly padded result. Use a scratch register if we need to do 3663 // a final bitcast, otherwise use the original result register. 3664 Register MergeDstReg; 3665 if (IsNarrow && DstTy.isVector()) 3666 MergeDstReg = MRI.createGenericVirtualRegister(DstScalarTy); 3667 else 3668 MergeDstReg = DstReg; 3669 3670 buildWidenedRemergeToDst(MergeDstReg, DstLCMTy, ResultRegs); 3671 3672 // Recast to vector if we narrowed a vector 3673 if (IsNarrow && DstTy.isVector()) 3674 MIRBuilder.buildBitcast(DstReg, MergeDstReg); 3675 3676 MI.eraseFromParent(); 3677 return Legalized; 3678 } 3679 3680 LegalizerHelper::LegalizeResult 3681 LegalizerHelper::fewerElementsVectorSextInReg(MachineInstr &MI, unsigned TypeIdx, 3682 LLT NarrowTy) { 3683 Register DstReg = MI.getOperand(0).getReg(); 3684 Register SrcReg = MI.getOperand(1).getReg(); 3685 int64_t Imm = MI.getOperand(2).getImm(); 3686 3687 LLT DstTy = MRI.getType(DstReg); 3688 3689 SmallVector<Register, 8> Parts; 3690 LLT GCDTy = extractGCDType(Parts, DstTy, NarrowTy, SrcReg); 3691 LLT LCMTy = buildLCMMergePieces(DstTy, NarrowTy, GCDTy, Parts); 3692 3693 for (Register &R : Parts) 3694 R = MIRBuilder.buildSExtInReg(NarrowTy, R, Imm).getReg(0); 3695 3696 buildWidenedRemergeToDst(DstReg, LCMTy, Parts); 3697 3698 MI.eraseFromParent(); 3699 return Legalized; 3700 } 3701 3702 LegalizerHelper::LegalizeResult 3703 LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx, 3704 LLT NarrowTy) { 3705 using namespace TargetOpcode; 3706 3707 switch (MI.getOpcode()) { 3708 case G_IMPLICIT_DEF: 3709 return fewerElementsVectorImplicitDef(MI, TypeIdx, NarrowTy); 3710 case G_TRUNC: 3711 case G_AND: 3712 case G_OR: 3713 case G_XOR: 3714 case G_ADD: 3715 case G_SUB: 3716 case G_MUL: 3717 case G_PTR_ADD: 3718 case G_SMULH: 3719 case G_UMULH: 3720 case G_FADD: 3721 case G_FMUL: 3722 case G_FSUB: 3723 case G_FNEG: 3724 case G_FABS: 3725 case G_FCANONICALIZE: 3726 case G_FDIV: 3727 case G_FREM: 3728 case G_FMA: 3729 case G_FMAD: 3730 case G_FPOW: 3731 case G_FEXP: 3732 case G_FEXP2: 3733 case G_FLOG: 3734 case G_FLOG2: 3735 case G_FLOG10: 3736 case G_FNEARBYINT: 3737 case G_FCEIL: 3738 case G_FFLOOR: 3739 case G_FRINT: 3740 case G_INTRINSIC_ROUND: 3741 case G_INTRINSIC_ROUNDEVEN: 3742 case G_INTRINSIC_TRUNC: 3743 case G_FCOS: 3744 case G_FSIN: 3745 case G_FSQRT: 3746 case G_BSWAP: 3747 case G_BITREVERSE: 3748 case G_SDIV: 3749 case G_UDIV: 3750 case G_SREM: 3751 case G_UREM: 3752 case G_SMIN: 3753 case G_SMAX: 3754 case G_UMIN: 3755 case G_UMAX: 3756 case G_FMINNUM: 3757 case G_FMAXNUM: 3758 case G_FMINNUM_IEEE: 3759 case G_FMAXNUM_IEEE: 3760 case G_FMINIMUM: 3761 case G_FMAXIMUM: 3762 case G_FSHL: 3763 case G_FSHR: 3764 case G_FREEZE: 3765 case G_SADDSAT: 3766 case G_SSUBSAT: 3767 case G_UADDSAT: 3768 case G_USUBSAT: 3769 return reduceOperationWidth(MI, TypeIdx, NarrowTy); 3770 case G_SHL: 3771 case G_LSHR: 3772 case G_ASHR: 3773 case G_CTLZ: 3774 case G_CTLZ_ZERO_UNDEF: 3775 case G_CTTZ: 3776 case G_CTTZ_ZERO_UNDEF: 3777 case G_CTPOP: 3778 case G_FCOPYSIGN: 3779 return fewerElementsVectorMultiEltType(MI, TypeIdx, NarrowTy); 3780 case G_ZEXT: 3781 case G_SEXT: 3782 case G_ANYEXT: 3783 case G_FPEXT: 3784 case G_FPTRUNC: 3785 case G_SITOFP: 3786 case G_UITOFP: 3787 case G_FPTOSI: 3788 case G_FPTOUI: 3789 case G_INTTOPTR: 3790 case G_PTRTOINT: 3791 case G_ADDRSPACE_CAST: 3792 return fewerElementsVectorCasts(MI, TypeIdx, NarrowTy); 3793 case G_ICMP: 3794 case G_FCMP: 3795 return fewerElementsVectorCmp(MI, TypeIdx, NarrowTy); 3796 case G_SELECT: 3797 return fewerElementsVectorSelect(MI, TypeIdx, NarrowTy); 3798 case G_PHI: 3799 return fewerElementsVectorPhi(MI, TypeIdx, NarrowTy); 3800 case G_UNMERGE_VALUES: 3801 return fewerElementsVectorUnmergeValues(MI, TypeIdx, NarrowTy); 3802 case G_BUILD_VECTOR: 3803 return fewerElementsVectorBuildVector(MI, TypeIdx, NarrowTy); 3804 case G_LOAD: 3805 case G_STORE: 3806 return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy); 3807 case G_SEXT_INREG: 3808 return fewerElementsVectorSextInReg(MI, TypeIdx, NarrowTy); 3809 default: 3810 return UnableToLegalize; 3811 } 3812 } 3813 3814 LegalizerHelper::LegalizeResult 3815 LegalizerHelper::narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt, 3816 const LLT HalfTy, const LLT AmtTy) { 3817 3818 Register InL = MRI.createGenericVirtualRegister(HalfTy); 3819 Register InH = MRI.createGenericVirtualRegister(HalfTy); 3820 MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1)); 3821 3822 if (Amt.isNullValue()) { 3823 MIRBuilder.buildMerge(MI.getOperand(0), {InL, InH}); 3824 MI.eraseFromParent(); 3825 return Legalized; 3826 } 3827 3828 LLT NVT = HalfTy; 3829 unsigned NVTBits = HalfTy.getSizeInBits(); 3830 unsigned VTBits = 2 * NVTBits; 3831 3832 SrcOp Lo(Register(0)), Hi(Register(0)); 3833 if (MI.getOpcode() == TargetOpcode::G_SHL) { 3834 if (Amt.ugt(VTBits)) { 3835 Lo = Hi = MIRBuilder.buildConstant(NVT, 0); 3836 } else if (Amt.ugt(NVTBits)) { 3837 Lo = MIRBuilder.buildConstant(NVT, 0); 3838 Hi = MIRBuilder.buildShl(NVT, InL, 3839 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits)); 3840 } else if (Amt == NVTBits) { 3841 Lo = MIRBuilder.buildConstant(NVT, 0); 3842 Hi = InL; 3843 } else { 3844 Lo = MIRBuilder.buildShl(NVT, InL, MIRBuilder.buildConstant(AmtTy, Amt)); 3845 auto OrLHS = 3846 MIRBuilder.buildShl(NVT, InH, MIRBuilder.buildConstant(AmtTy, Amt)); 3847 auto OrRHS = MIRBuilder.buildLShr( 3848 NVT, InL, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits)); 3849 Hi = MIRBuilder.buildOr(NVT, OrLHS, OrRHS); 3850 } 3851 } else if (MI.getOpcode() == TargetOpcode::G_LSHR) { 3852 if (Amt.ugt(VTBits)) { 3853 Lo = Hi = MIRBuilder.buildConstant(NVT, 0); 3854 } else if (Amt.ugt(NVTBits)) { 3855 Lo = MIRBuilder.buildLShr(NVT, InH, 3856 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits)); 3857 Hi = MIRBuilder.buildConstant(NVT, 0); 3858 } else if (Amt == NVTBits) { 3859 Lo = InH; 3860 Hi = MIRBuilder.buildConstant(NVT, 0); 3861 } else { 3862 auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt); 3863 3864 auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst); 3865 auto OrRHS = MIRBuilder.buildShl( 3866 NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits)); 3867 3868 Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS); 3869 Hi = MIRBuilder.buildLShr(NVT, InH, ShiftAmtConst); 3870 } 3871 } else { 3872 if (Amt.ugt(VTBits)) { 3873 Hi = Lo = MIRBuilder.buildAShr( 3874 NVT, InH, MIRBuilder.buildConstant(AmtTy, NVTBits - 1)); 3875 } else if (Amt.ugt(NVTBits)) { 3876 Lo = MIRBuilder.buildAShr(NVT, InH, 3877 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits)); 3878 Hi = MIRBuilder.buildAShr(NVT, InH, 3879 MIRBuilder.buildConstant(AmtTy, NVTBits - 1)); 3880 } else if (Amt == NVTBits) { 3881 Lo = InH; 3882 Hi = MIRBuilder.buildAShr(NVT, InH, 3883 MIRBuilder.buildConstant(AmtTy, NVTBits - 1)); 3884 } else { 3885 auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt); 3886 3887 auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst); 3888 auto OrRHS = MIRBuilder.buildShl( 3889 NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits)); 3890 3891 Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS); 3892 Hi = MIRBuilder.buildAShr(NVT, InH, ShiftAmtConst); 3893 } 3894 } 3895 3896 MIRBuilder.buildMerge(MI.getOperand(0), {Lo, Hi}); 3897 MI.eraseFromParent(); 3898 3899 return Legalized; 3900 } 3901 3902 // TODO: Optimize if constant shift amount. 3903 LegalizerHelper::LegalizeResult 3904 LegalizerHelper::narrowScalarShift(MachineInstr &MI, unsigned TypeIdx, 3905 LLT RequestedTy) { 3906 if (TypeIdx == 1) { 3907 Observer.changingInstr(MI); 3908 narrowScalarSrc(MI, RequestedTy, 2); 3909 Observer.changedInstr(MI); 3910 return Legalized; 3911 } 3912 3913 Register DstReg = MI.getOperand(0).getReg(); 3914 LLT DstTy = MRI.getType(DstReg); 3915 if (DstTy.isVector()) 3916 return UnableToLegalize; 3917 3918 Register Amt = MI.getOperand(2).getReg(); 3919 LLT ShiftAmtTy = MRI.getType(Amt); 3920 const unsigned DstEltSize = DstTy.getScalarSizeInBits(); 3921 if (DstEltSize % 2 != 0) 3922 return UnableToLegalize; 3923 3924 // Ignore the input type. We can only go to exactly half the size of the 3925 // input. If that isn't small enough, the resulting pieces will be further 3926 // legalized. 3927 const unsigned NewBitSize = DstEltSize / 2; 3928 const LLT HalfTy = LLT::scalar(NewBitSize); 3929 const LLT CondTy = LLT::scalar(1); 3930 3931 if (const MachineInstr *KShiftAmt = 3932 getOpcodeDef(TargetOpcode::G_CONSTANT, Amt, MRI)) { 3933 return narrowScalarShiftByConstant( 3934 MI, KShiftAmt->getOperand(1).getCImm()->getValue(), HalfTy, ShiftAmtTy); 3935 } 3936 3937 // TODO: Expand with known bits. 3938 3939 // Handle the fully general expansion by an unknown amount. 3940 auto NewBits = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize); 3941 3942 Register InL = MRI.createGenericVirtualRegister(HalfTy); 3943 Register InH = MRI.createGenericVirtualRegister(HalfTy); 3944 MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1)); 3945 3946 auto AmtExcess = MIRBuilder.buildSub(ShiftAmtTy, Amt, NewBits); 3947 auto AmtLack = MIRBuilder.buildSub(ShiftAmtTy, NewBits, Amt); 3948 3949 auto Zero = MIRBuilder.buildConstant(ShiftAmtTy, 0); 3950 auto IsShort = MIRBuilder.buildICmp(ICmpInst::ICMP_ULT, CondTy, Amt, NewBits); 3951 auto IsZero = MIRBuilder.buildICmp(ICmpInst::ICMP_EQ, CondTy, Amt, Zero); 3952 3953 Register ResultRegs[2]; 3954 switch (MI.getOpcode()) { 3955 case TargetOpcode::G_SHL: { 3956 // Short: ShAmt < NewBitSize 3957 auto LoS = MIRBuilder.buildShl(HalfTy, InL, Amt); 3958 3959 auto LoOr = MIRBuilder.buildLShr(HalfTy, InL, AmtLack); 3960 auto HiOr = MIRBuilder.buildShl(HalfTy, InH, Amt); 3961 auto HiS = MIRBuilder.buildOr(HalfTy, LoOr, HiOr); 3962 3963 // Long: ShAmt >= NewBitSize 3964 auto LoL = MIRBuilder.buildConstant(HalfTy, 0); // Lo part is zero. 3965 auto HiL = MIRBuilder.buildShl(HalfTy, InL, AmtExcess); // Hi from Lo part. 3966 3967 auto Lo = MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL); 3968 auto Hi = MIRBuilder.buildSelect( 3969 HalfTy, IsZero, InH, MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL)); 3970 3971 ResultRegs[0] = Lo.getReg(0); 3972 ResultRegs[1] = Hi.getReg(0); 3973 break; 3974 } 3975 case TargetOpcode::G_LSHR: 3976 case TargetOpcode::G_ASHR: { 3977 // Short: ShAmt < NewBitSize 3978 auto HiS = MIRBuilder.buildInstr(MI.getOpcode(), {HalfTy}, {InH, Amt}); 3979 3980 auto LoOr = MIRBuilder.buildLShr(HalfTy, InL, Amt); 3981 auto HiOr = MIRBuilder.buildShl(HalfTy, InH, AmtLack); 3982 auto LoS = MIRBuilder.buildOr(HalfTy, LoOr, HiOr); 3983 3984 // Long: ShAmt >= NewBitSize 3985 MachineInstrBuilder HiL; 3986 if (MI.getOpcode() == TargetOpcode::G_LSHR) { 3987 HiL = MIRBuilder.buildConstant(HalfTy, 0); // Hi part is zero. 3988 } else { 3989 auto ShiftAmt = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize - 1); 3990 HiL = MIRBuilder.buildAShr(HalfTy, InH, ShiftAmt); // Sign of Hi part. 3991 } 3992 auto LoL = MIRBuilder.buildInstr(MI.getOpcode(), {HalfTy}, 3993 {InH, AmtExcess}); // Lo from Hi part. 3994 3995 auto Lo = MIRBuilder.buildSelect( 3996 HalfTy, IsZero, InL, MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL)); 3997 3998 auto Hi = MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL); 3999 4000 ResultRegs[0] = Lo.getReg(0); 4001 ResultRegs[1] = Hi.getReg(0); 4002 break; 4003 } 4004 default: 4005 llvm_unreachable("not a shift"); 4006 } 4007 4008 MIRBuilder.buildMerge(DstReg, ResultRegs); 4009 MI.eraseFromParent(); 4010 return Legalized; 4011 } 4012 4013 LegalizerHelper::LegalizeResult 4014 LegalizerHelper::moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx, 4015 LLT MoreTy) { 4016 assert(TypeIdx == 0 && "Expecting only Idx 0"); 4017 4018 Observer.changingInstr(MI); 4019 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) { 4020 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB(); 4021 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator()); 4022 moreElementsVectorSrc(MI, MoreTy, I); 4023 } 4024 4025 MachineBasicBlock &MBB = *MI.getParent(); 4026 MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI()); 4027 moreElementsVectorDst(MI, MoreTy, 0); 4028 Observer.changedInstr(MI); 4029 return Legalized; 4030 } 4031 4032 LegalizerHelper::LegalizeResult 4033 LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx, 4034 LLT MoreTy) { 4035 unsigned Opc = MI.getOpcode(); 4036 switch (Opc) { 4037 case TargetOpcode::G_IMPLICIT_DEF: 4038 case TargetOpcode::G_LOAD: { 4039 if (TypeIdx != 0) 4040 return UnableToLegalize; 4041 Observer.changingInstr(MI); 4042 moreElementsVectorDst(MI, MoreTy, 0); 4043 Observer.changedInstr(MI); 4044 return Legalized; 4045 } 4046 case TargetOpcode::G_STORE: 4047 if (TypeIdx != 0) 4048 return UnableToLegalize; 4049 Observer.changingInstr(MI); 4050 moreElementsVectorSrc(MI, MoreTy, 0); 4051 Observer.changedInstr(MI); 4052 return Legalized; 4053 case TargetOpcode::G_AND: 4054 case TargetOpcode::G_OR: 4055 case TargetOpcode::G_XOR: 4056 case TargetOpcode::G_SMIN: 4057 case TargetOpcode::G_SMAX: 4058 case TargetOpcode::G_UMIN: 4059 case TargetOpcode::G_UMAX: 4060 case TargetOpcode::G_FMINNUM: 4061 case TargetOpcode::G_FMAXNUM: 4062 case TargetOpcode::G_FMINNUM_IEEE: 4063 case TargetOpcode::G_FMAXNUM_IEEE: 4064 case TargetOpcode::G_FMINIMUM: 4065 case TargetOpcode::G_FMAXIMUM: { 4066 Observer.changingInstr(MI); 4067 moreElementsVectorSrc(MI, MoreTy, 1); 4068 moreElementsVectorSrc(MI, MoreTy, 2); 4069 moreElementsVectorDst(MI, MoreTy, 0); 4070 Observer.changedInstr(MI); 4071 return Legalized; 4072 } 4073 case TargetOpcode::G_EXTRACT: 4074 if (TypeIdx != 1) 4075 return UnableToLegalize; 4076 Observer.changingInstr(MI); 4077 moreElementsVectorSrc(MI, MoreTy, 1); 4078 Observer.changedInstr(MI); 4079 return Legalized; 4080 case TargetOpcode::G_INSERT: 4081 case TargetOpcode::G_FREEZE: 4082 if (TypeIdx != 0) 4083 return UnableToLegalize; 4084 Observer.changingInstr(MI); 4085 moreElementsVectorSrc(MI, MoreTy, 1); 4086 moreElementsVectorDst(MI, MoreTy, 0); 4087 Observer.changedInstr(MI); 4088 return Legalized; 4089 case TargetOpcode::G_SELECT: 4090 if (TypeIdx != 0) 4091 return UnableToLegalize; 4092 if (MRI.getType(MI.getOperand(1).getReg()).isVector()) 4093 return UnableToLegalize; 4094 4095 Observer.changingInstr(MI); 4096 moreElementsVectorSrc(MI, MoreTy, 2); 4097 moreElementsVectorSrc(MI, MoreTy, 3); 4098 moreElementsVectorDst(MI, MoreTy, 0); 4099 Observer.changedInstr(MI); 4100 return Legalized; 4101 case TargetOpcode::G_UNMERGE_VALUES: { 4102 if (TypeIdx != 1) 4103 return UnableToLegalize; 4104 4105 LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); 4106 int NumDst = MI.getNumOperands() - 1; 4107 moreElementsVectorSrc(MI, MoreTy, NumDst); 4108 4109 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES); 4110 for (int I = 0; I != NumDst; ++I) 4111 MIB.addDef(MI.getOperand(I).getReg()); 4112 4113 int NewNumDst = MoreTy.getSizeInBits() / DstTy.getSizeInBits(); 4114 for (int I = NumDst; I != NewNumDst; ++I) 4115 MIB.addDef(MRI.createGenericVirtualRegister(DstTy)); 4116 4117 MIB.addUse(MI.getOperand(NumDst).getReg()); 4118 MI.eraseFromParent(); 4119 return Legalized; 4120 } 4121 case TargetOpcode::G_PHI: 4122 return moreElementsVectorPhi(MI, TypeIdx, MoreTy); 4123 default: 4124 return UnableToLegalize; 4125 } 4126 } 4127 4128 void LegalizerHelper::multiplyRegisters(SmallVectorImpl<Register> &DstRegs, 4129 ArrayRef<Register> Src1Regs, 4130 ArrayRef<Register> Src2Regs, 4131 LLT NarrowTy) { 4132 MachineIRBuilder &B = MIRBuilder; 4133 unsigned SrcParts = Src1Regs.size(); 4134 unsigned DstParts = DstRegs.size(); 4135 4136 unsigned DstIdx = 0; // Low bits of the result. 4137 Register FactorSum = 4138 B.buildMul(NarrowTy, Src1Regs[DstIdx], Src2Regs[DstIdx]).getReg(0); 4139 DstRegs[DstIdx] = FactorSum; 4140 4141 unsigned CarrySumPrevDstIdx; 4142 SmallVector<Register, 4> Factors; 4143 4144 for (DstIdx = 1; DstIdx < DstParts; DstIdx++) { 4145 // Collect low parts of muls for DstIdx. 4146 for (unsigned i = DstIdx + 1 < SrcParts ? 0 : DstIdx - SrcParts + 1; 4147 i <= std::min(DstIdx, SrcParts - 1); ++i) { 4148 MachineInstrBuilder Mul = 4149 B.buildMul(NarrowTy, Src1Regs[DstIdx - i], Src2Regs[i]); 4150 Factors.push_back(Mul.getReg(0)); 4151 } 4152 // Collect high parts of muls from previous DstIdx. 4153 for (unsigned i = DstIdx < SrcParts ? 0 : DstIdx - SrcParts; 4154 i <= std::min(DstIdx - 1, SrcParts - 1); ++i) { 4155 MachineInstrBuilder Umulh = 4156 B.buildUMulH(NarrowTy, Src1Regs[DstIdx - 1 - i], Src2Regs[i]); 4157 Factors.push_back(Umulh.getReg(0)); 4158 } 4159 // Add CarrySum from additions calculated for previous DstIdx. 4160 if (DstIdx != 1) { 4161 Factors.push_back(CarrySumPrevDstIdx); 4162 } 4163 4164 Register CarrySum; 4165 // Add all factors and accumulate all carries into CarrySum. 4166 if (DstIdx != DstParts - 1) { 4167 MachineInstrBuilder Uaddo = 4168 B.buildUAddo(NarrowTy, LLT::scalar(1), Factors[0], Factors[1]); 4169 FactorSum = Uaddo.getReg(0); 4170 CarrySum = B.buildZExt(NarrowTy, Uaddo.getReg(1)).getReg(0); 4171 for (unsigned i = 2; i < Factors.size(); ++i) { 4172 MachineInstrBuilder Uaddo = 4173 B.buildUAddo(NarrowTy, LLT::scalar(1), FactorSum, Factors[i]); 4174 FactorSum = Uaddo.getReg(0); 4175 MachineInstrBuilder Carry = B.buildZExt(NarrowTy, Uaddo.getReg(1)); 4176 CarrySum = B.buildAdd(NarrowTy, CarrySum, Carry).getReg(0); 4177 } 4178 } else { 4179 // Since value for the next index is not calculated, neither is CarrySum. 4180 FactorSum = B.buildAdd(NarrowTy, Factors[0], Factors[1]).getReg(0); 4181 for (unsigned i = 2; i < Factors.size(); ++i) 4182 FactorSum = B.buildAdd(NarrowTy, FactorSum, Factors[i]).getReg(0); 4183 } 4184 4185 CarrySumPrevDstIdx = CarrySum; 4186 DstRegs[DstIdx] = FactorSum; 4187 Factors.clear(); 4188 } 4189 } 4190 4191 LegalizerHelper::LegalizeResult 4192 LegalizerHelper::narrowScalarMul(MachineInstr &MI, LLT NarrowTy) { 4193 Register DstReg = MI.getOperand(0).getReg(); 4194 Register Src1 = MI.getOperand(1).getReg(); 4195 Register Src2 = MI.getOperand(2).getReg(); 4196 4197 LLT Ty = MRI.getType(DstReg); 4198 if (Ty.isVector()) 4199 return UnableToLegalize; 4200 4201 unsigned SrcSize = MRI.getType(Src1).getSizeInBits(); 4202 unsigned DstSize = Ty.getSizeInBits(); 4203 unsigned NarrowSize = NarrowTy.getSizeInBits(); 4204 if (DstSize % NarrowSize != 0 || SrcSize % NarrowSize != 0) 4205 return UnableToLegalize; 4206 4207 unsigned NumDstParts = DstSize / NarrowSize; 4208 unsigned NumSrcParts = SrcSize / NarrowSize; 4209 bool IsMulHigh = MI.getOpcode() == TargetOpcode::G_UMULH; 4210 unsigned DstTmpParts = NumDstParts * (IsMulHigh ? 2 : 1); 4211 4212 SmallVector<Register, 2> Src1Parts, Src2Parts; 4213 SmallVector<Register, 2> DstTmpRegs(DstTmpParts); 4214 extractParts(Src1, NarrowTy, NumSrcParts, Src1Parts); 4215 extractParts(Src2, NarrowTy, NumSrcParts, Src2Parts); 4216 multiplyRegisters(DstTmpRegs, Src1Parts, Src2Parts, NarrowTy); 4217 4218 // Take only high half of registers if this is high mul. 4219 ArrayRef<Register> DstRegs( 4220 IsMulHigh ? &DstTmpRegs[DstTmpParts / 2] : &DstTmpRegs[0], NumDstParts); 4221 MIRBuilder.buildMerge(DstReg, DstRegs); 4222 MI.eraseFromParent(); 4223 return Legalized; 4224 } 4225 4226 LegalizerHelper::LegalizeResult 4227 LegalizerHelper::narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx, 4228 LLT NarrowTy) { 4229 if (TypeIdx != 1) 4230 return UnableToLegalize; 4231 4232 uint64_t NarrowSize = NarrowTy.getSizeInBits(); 4233 4234 int64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits(); 4235 // FIXME: add support for when SizeOp1 isn't an exact multiple of 4236 // NarrowSize. 4237 if (SizeOp1 % NarrowSize != 0) 4238 return UnableToLegalize; 4239 int NumParts = SizeOp1 / NarrowSize; 4240 4241 SmallVector<Register, 2> SrcRegs, DstRegs; 4242 SmallVector<uint64_t, 2> Indexes; 4243 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs); 4244 4245 Register OpReg = MI.getOperand(0).getReg(); 4246 uint64_t OpStart = MI.getOperand(2).getImm(); 4247 uint64_t OpSize = MRI.getType(OpReg).getSizeInBits(); 4248 for (int i = 0; i < NumParts; ++i) { 4249 unsigned SrcStart = i * NarrowSize; 4250 4251 if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) { 4252 // No part of the extract uses this subregister, ignore it. 4253 continue; 4254 } else if (SrcStart == OpStart && NarrowTy == MRI.getType(OpReg)) { 4255 // The entire subregister is extracted, forward the value. 4256 DstRegs.push_back(SrcRegs[i]); 4257 continue; 4258 } 4259 4260 // OpSegStart is where this destination segment would start in OpReg if it 4261 // extended infinitely in both directions. 4262 int64_t ExtractOffset; 4263 uint64_t SegSize; 4264 if (OpStart < SrcStart) { 4265 ExtractOffset = 0; 4266 SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart); 4267 } else { 4268 ExtractOffset = OpStart - SrcStart; 4269 SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize); 4270 } 4271 4272 Register SegReg = SrcRegs[i]; 4273 if (ExtractOffset != 0 || SegSize != NarrowSize) { 4274 // A genuine extract is needed. 4275 SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize)); 4276 MIRBuilder.buildExtract(SegReg, SrcRegs[i], ExtractOffset); 4277 } 4278 4279 DstRegs.push_back(SegReg); 4280 } 4281 4282 Register DstReg = MI.getOperand(0).getReg(); 4283 if (MRI.getType(DstReg).isVector()) 4284 MIRBuilder.buildBuildVector(DstReg, DstRegs); 4285 else if (DstRegs.size() > 1) 4286 MIRBuilder.buildMerge(DstReg, DstRegs); 4287 else 4288 MIRBuilder.buildCopy(DstReg, DstRegs[0]); 4289 MI.eraseFromParent(); 4290 return Legalized; 4291 } 4292 4293 LegalizerHelper::LegalizeResult 4294 LegalizerHelper::narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx, 4295 LLT NarrowTy) { 4296 // FIXME: Don't know how to handle secondary types yet. 4297 if (TypeIdx != 0) 4298 return UnableToLegalize; 4299 4300 uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 4301 uint64_t NarrowSize = NarrowTy.getSizeInBits(); 4302 4303 // FIXME: add support for when SizeOp0 isn't an exact multiple of 4304 // NarrowSize. 4305 if (SizeOp0 % NarrowSize != 0) 4306 return UnableToLegalize; 4307 4308 int NumParts = SizeOp0 / NarrowSize; 4309 4310 SmallVector<Register, 2> SrcRegs, DstRegs; 4311 SmallVector<uint64_t, 2> Indexes; 4312 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs); 4313 4314 Register OpReg = MI.getOperand(2).getReg(); 4315 uint64_t OpStart = MI.getOperand(3).getImm(); 4316 uint64_t OpSize = MRI.getType(OpReg).getSizeInBits(); 4317 for (int i = 0; i < NumParts; ++i) { 4318 unsigned DstStart = i * NarrowSize; 4319 4320 if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) { 4321 // No part of the insert affects this subregister, forward the original. 4322 DstRegs.push_back(SrcRegs[i]); 4323 continue; 4324 } else if (DstStart == OpStart && NarrowTy == MRI.getType(OpReg)) { 4325 // The entire subregister is defined by this insert, forward the new 4326 // value. 4327 DstRegs.push_back(OpReg); 4328 continue; 4329 } 4330 4331 // OpSegStart is where this destination segment would start in OpReg if it 4332 // extended infinitely in both directions. 4333 int64_t ExtractOffset, InsertOffset; 4334 uint64_t SegSize; 4335 if (OpStart < DstStart) { 4336 InsertOffset = 0; 4337 ExtractOffset = DstStart - OpStart; 4338 SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart); 4339 } else { 4340 InsertOffset = OpStart - DstStart; 4341 ExtractOffset = 0; 4342 SegSize = 4343 std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart); 4344 } 4345 4346 Register SegReg = OpReg; 4347 if (ExtractOffset != 0 || SegSize != OpSize) { 4348 // A genuine extract is needed. 4349 SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize)); 4350 MIRBuilder.buildExtract(SegReg, OpReg, ExtractOffset); 4351 } 4352 4353 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy); 4354 MIRBuilder.buildInsert(DstReg, SrcRegs[i], SegReg, InsertOffset); 4355 DstRegs.push_back(DstReg); 4356 } 4357 4358 assert(DstRegs.size() == (unsigned)NumParts && "not all parts covered"); 4359 Register DstReg = MI.getOperand(0).getReg(); 4360 if(MRI.getType(DstReg).isVector()) 4361 MIRBuilder.buildBuildVector(DstReg, DstRegs); 4362 else 4363 MIRBuilder.buildMerge(DstReg, DstRegs); 4364 MI.eraseFromParent(); 4365 return Legalized; 4366 } 4367 4368 LegalizerHelper::LegalizeResult 4369 LegalizerHelper::narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx, 4370 LLT NarrowTy) { 4371 Register DstReg = MI.getOperand(0).getReg(); 4372 LLT DstTy = MRI.getType(DstReg); 4373 4374 assert(MI.getNumOperands() == 3 && TypeIdx == 0); 4375 4376 SmallVector<Register, 4> DstRegs, DstLeftoverRegs; 4377 SmallVector<Register, 4> Src0Regs, Src0LeftoverRegs; 4378 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs; 4379 LLT LeftoverTy; 4380 if (!extractParts(MI.getOperand(1).getReg(), DstTy, NarrowTy, LeftoverTy, 4381 Src0Regs, Src0LeftoverRegs)) 4382 return UnableToLegalize; 4383 4384 LLT Unused; 4385 if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, Unused, 4386 Src1Regs, Src1LeftoverRegs)) 4387 llvm_unreachable("inconsistent extractParts result"); 4388 4389 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) { 4390 auto Inst = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy}, 4391 {Src0Regs[I], Src1Regs[I]}); 4392 DstRegs.push_back(Inst.getReg(0)); 4393 } 4394 4395 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) { 4396 auto Inst = MIRBuilder.buildInstr( 4397 MI.getOpcode(), 4398 {LeftoverTy}, {Src0LeftoverRegs[I], Src1LeftoverRegs[I]}); 4399 DstLeftoverRegs.push_back(Inst.getReg(0)); 4400 } 4401 4402 insertParts(DstReg, DstTy, NarrowTy, DstRegs, 4403 LeftoverTy, DstLeftoverRegs); 4404 4405 MI.eraseFromParent(); 4406 return Legalized; 4407 } 4408 4409 LegalizerHelper::LegalizeResult 4410 LegalizerHelper::narrowScalarExt(MachineInstr &MI, unsigned TypeIdx, 4411 LLT NarrowTy) { 4412 if (TypeIdx != 0) 4413 return UnableToLegalize; 4414 4415 Register DstReg = MI.getOperand(0).getReg(); 4416 Register SrcReg = MI.getOperand(1).getReg(); 4417 4418 LLT DstTy = MRI.getType(DstReg); 4419 if (DstTy.isVector()) 4420 return UnableToLegalize; 4421 4422 SmallVector<Register, 8> Parts; 4423 LLT GCDTy = extractGCDType(Parts, DstTy, NarrowTy, SrcReg); 4424 LLT LCMTy = buildLCMMergePieces(DstTy, NarrowTy, GCDTy, Parts, MI.getOpcode()); 4425 buildWidenedRemergeToDst(DstReg, LCMTy, Parts); 4426 4427 MI.eraseFromParent(); 4428 return Legalized; 4429 } 4430 4431 LegalizerHelper::LegalizeResult 4432 LegalizerHelper::narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx, 4433 LLT NarrowTy) { 4434 if (TypeIdx != 0) 4435 return UnableToLegalize; 4436 4437 Register CondReg = MI.getOperand(1).getReg(); 4438 LLT CondTy = MRI.getType(CondReg); 4439 if (CondTy.isVector()) // TODO: Handle vselect 4440 return UnableToLegalize; 4441 4442 Register DstReg = MI.getOperand(0).getReg(); 4443 LLT DstTy = MRI.getType(DstReg); 4444 4445 SmallVector<Register, 4> DstRegs, DstLeftoverRegs; 4446 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs; 4447 SmallVector<Register, 4> Src2Regs, Src2LeftoverRegs; 4448 LLT LeftoverTy; 4449 if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, LeftoverTy, 4450 Src1Regs, Src1LeftoverRegs)) 4451 return UnableToLegalize; 4452 4453 LLT Unused; 4454 if (!extractParts(MI.getOperand(3).getReg(), DstTy, NarrowTy, Unused, 4455 Src2Regs, Src2LeftoverRegs)) 4456 llvm_unreachable("inconsistent extractParts result"); 4457 4458 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) { 4459 auto Select = MIRBuilder.buildSelect(NarrowTy, 4460 CondReg, Src1Regs[I], Src2Regs[I]); 4461 DstRegs.push_back(Select.getReg(0)); 4462 } 4463 4464 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) { 4465 auto Select = MIRBuilder.buildSelect( 4466 LeftoverTy, CondReg, Src1LeftoverRegs[I], Src2LeftoverRegs[I]); 4467 DstLeftoverRegs.push_back(Select.getReg(0)); 4468 } 4469 4470 insertParts(DstReg, DstTy, NarrowTy, DstRegs, 4471 LeftoverTy, DstLeftoverRegs); 4472 4473 MI.eraseFromParent(); 4474 return Legalized; 4475 } 4476 4477 LegalizerHelper::LegalizeResult 4478 LegalizerHelper::narrowScalarCTLZ(MachineInstr &MI, unsigned TypeIdx, 4479 LLT NarrowTy) { 4480 if (TypeIdx != 1) 4481 return UnableToLegalize; 4482 4483 Register DstReg = MI.getOperand(0).getReg(); 4484 Register SrcReg = MI.getOperand(1).getReg(); 4485 LLT DstTy = MRI.getType(DstReg); 4486 LLT SrcTy = MRI.getType(SrcReg); 4487 unsigned NarrowSize = NarrowTy.getSizeInBits(); 4488 4489 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) { 4490 const bool IsUndef = MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF; 4491 4492 MachineIRBuilder &B = MIRBuilder; 4493 auto UnmergeSrc = B.buildUnmerge(NarrowTy, SrcReg); 4494 // ctlz(Hi:Lo) -> Hi == 0 ? (NarrowSize + ctlz(Lo)) : ctlz(Hi) 4495 auto C_0 = B.buildConstant(NarrowTy, 0); 4496 auto HiIsZero = B.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1), 4497 UnmergeSrc.getReg(1), C_0); 4498 auto LoCTLZ = IsUndef ? 4499 B.buildCTLZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(0)) : 4500 B.buildCTLZ(DstTy, UnmergeSrc.getReg(0)); 4501 auto C_NarrowSize = B.buildConstant(DstTy, NarrowSize); 4502 auto HiIsZeroCTLZ = B.buildAdd(DstTy, LoCTLZ, C_NarrowSize); 4503 auto HiCTLZ = B.buildCTLZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(1)); 4504 B.buildSelect(DstReg, HiIsZero, HiIsZeroCTLZ, HiCTLZ); 4505 4506 MI.eraseFromParent(); 4507 return Legalized; 4508 } 4509 4510 return UnableToLegalize; 4511 } 4512 4513 LegalizerHelper::LegalizeResult 4514 LegalizerHelper::narrowScalarCTTZ(MachineInstr &MI, unsigned TypeIdx, 4515 LLT NarrowTy) { 4516 if (TypeIdx != 1) 4517 return UnableToLegalize; 4518 4519 Register DstReg = MI.getOperand(0).getReg(); 4520 Register SrcReg = MI.getOperand(1).getReg(); 4521 LLT DstTy = MRI.getType(DstReg); 4522 LLT SrcTy = MRI.getType(SrcReg); 4523 unsigned NarrowSize = NarrowTy.getSizeInBits(); 4524 4525 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) { 4526 const bool IsUndef = MI.getOpcode() == TargetOpcode::G_CTTZ_ZERO_UNDEF; 4527 4528 MachineIRBuilder &B = MIRBuilder; 4529 auto UnmergeSrc = B.buildUnmerge(NarrowTy, SrcReg); 4530 // cttz(Hi:Lo) -> Lo == 0 ? (cttz(Hi) + NarrowSize) : cttz(Lo) 4531 auto C_0 = B.buildConstant(NarrowTy, 0); 4532 auto LoIsZero = B.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1), 4533 UnmergeSrc.getReg(0), C_0); 4534 auto HiCTTZ = IsUndef ? 4535 B.buildCTTZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(1)) : 4536 B.buildCTTZ(DstTy, UnmergeSrc.getReg(1)); 4537 auto C_NarrowSize = B.buildConstant(DstTy, NarrowSize); 4538 auto LoIsZeroCTTZ = B.buildAdd(DstTy, HiCTTZ, C_NarrowSize); 4539 auto LoCTTZ = B.buildCTTZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(0)); 4540 B.buildSelect(DstReg, LoIsZero, LoIsZeroCTTZ, LoCTTZ); 4541 4542 MI.eraseFromParent(); 4543 return Legalized; 4544 } 4545 4546 return UnableToLegalize; 4547 } 4548 4549 LegalizerHelper::LegalizeResult 4550 LegalizerHelper::narrowScalarCTPOP(MachineInstr &MI, unsigned TypeIdx, 4551 LLT NarrowTy) { 4552 if (TypeIdx != 1) 4553 return UnableToLegalize; 4554 4555 Register DstReg = MI.getOperand(0).getReg(); 4556 LLT DstTy = MRI.getType(DstReg); 4557 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg()); 4558 unsigned NarrowSize = NarrowTy.getSizeInBits(); 4559 4560 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) { 4561 auto UnmergeSrc = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1)); 4562 4563 auto LoCTPOP = MIRBuilder.buildCTPOP(DstTy, UnmergeSrc.getReg(0)); 4564 auto HiCTPOP = MIRBuilder.buildCTPOP(DstTy, UnmergeSrc.getReg(1)); 4565 MIRBuilder.buildAdd(DstReg, HiCTPOP, LoCTPOP); 4566 4567 MI.eraseFromParent(); 4568 return Legalized; 4569 } 4570 4571 return UnableToLegalize; 4572 } 4573 4574 LegalizerHelper::LegalizeResult 4575 LegalizerHelper::lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { 4576 unsigned Opc = MI.getOpcode(); 4577 const auto &TII = MIRBuilder.getTII(); 4578 auto isSupported = [this](const LegalityQuery &Q) { 4579 auto QAction = LI.getAction(Q).Action; 4580 return QAction == Legal || QAction == Libcall || QAction == Custom; 4581 }; 4582 switch (Opc) { 4583 default: 4584 return UnableToLegalize; 4585 case TargetOpcode::G_CTLZ_ZERO_UNDEF: { 4586 // This trivially expands to CTLZ. 4587 Observer.changingInstr(MI); 4588 MI.setDesc(TII.get(TargetOpcode::G_CTLZ)); 4589 Observer.changedInstr(MI); 4590 return Legalized; 4591 } 4592 case TargetOpcode::G_CTLZ: { 4593 Register DstReg = MI.getOperand(0).getReg(); 4594 Register SrcReg = MI.getOperand(1).getReg(); 4595 LLT DstTy = MRI.getType(DstReg); 4596 LLT SrcTy = MRI.getType(SrcReg); 4597 unsigned Len = SrcTy.getSizeInBits(); 4598 4599 if (isSupported({TargetOpcode::G_CTLZ_ZERO_UNDEF, {DstTy, SrcTy}})) { 4600 // If CTLZ_ZERO_UNDEF is supported, emit that and a select for zero. 4601 auto CtlzZU = MIRBuilder.buildCTLZ_ZERO_UNDEF(DstTy, SrcReg); 4602 auto ZeroSrc = MIRBuilder.buildConstant(SrcTy, 0); 4603 auto ICmp = MIRBuilder.buildICmp( 4604 CmpInst::ICMP_EQ, SrcTy.changeElementSize(1), SrcReg, ZeroSrc); 4605 auto LenConst = MIRBuilder.buildConstant(DstTy, Len); 4606 MIRBuilder.buildSelect(DstReg, ICmp, LenConst, CtlzZU); 4607 MI.eraseFromParent(); 4608 return Legalized; 4609 } 4610 // for now, we do this: 4611 // NewLen = NextPowerOf2(Len); 4612 // x = x | (x >> 1); 4613 // x = x | (x >> 2); 4614 // ... 4615 // x = x | (x >>16); 4616 // x = x | (x >>32); // for 64-bit input 4617 // Upto NewLen/2 4618 // return Len - popcount(x); 4619 // 4620 // Ref: "Hacker's Delight" by Henry Warren 4621 Register Op = SrcReg; 4622 unsigned NewLen = PowerOf2Ceil(Len); 4623 for (unsigned i = 0; (1U << i) <= (NewLen / 2); ++i) { 4624 auto MIBShiftAmt = MIRBuilder.buildConstant(SrcTy, 1ULL << i); 4625 auto MIBOp = MIRBuilder.buildOr( 4626 SrcTy, Op, MIRBuilder.buildLShr(SrcTy, Op, MIBShiftAmt)); 4627 Op = MIBOp.getReg(0); 4628 } 4629 auto MIBPop = MIRBuilder.buildCTPOP(DstTy, Op); 4630 MIRBuilder.buildSub(MI.getOperand(0), MIRBuilder.buildConstant(DstTy, Len), 4631 MIBPop); 4632 MI.eraseFromParent(); 4633 return Legalized; 4634 } 4635 case TargetOpcode::G_CTTZ_ZERO_UNDEF: { 4636 // This trivially expands to CTTZ. 4637 Observer.changingInstr(MI); 4638 MI.setDesc(TII.get(TargetOpcode::G_CTTZ)); 4639 Observer.changedInstr(MI); 4640 return Legalized; 4641 } 4642 case TargetOpcode::G_CTTZ: { 4643 Register DstReg = MI.getOperand(0).getReg(); 4644 Register SrcReg = MI.getOperand(1).getReg(); 4645 LLT DstTy = MRI.getType(DstReg); 4646 LLT SrcTy = MRI.getType(SrcReg); 4647 4648 unsigned Len = SrcTy.getSizeInBits(); 4649 if (isSupported({TargetOpcode::G_CTTZ_ZERO_UNDEF, {DstTy, SrcTy}})) { 4650 // If CTTZ_ZERO_UNDEF is legal or custom, emit that and a select with 4651 // zero. 4652 auto CttzZU = MIRBuilder.buildCTTZ_ZERO_UNDEF(DstTy, SrcReg); 4653 auto Zero = MIRBuilder.buildConstant(SrcTy, 0); 4654 auto ICmp = MIRBuilder.buildICmp( 4655 CmpInst::ICMP_EQ, DstTy.changeElementSize(1), SrcReg, Zero); 4656 auto LenConst = MIRBuilder.buildConstant(DstTy, Len); 4657 MIRBuilder.buildSelect(DstReg, ICmp, LenConst, CttzZU); 4658 MI.eraseFromParent(); 4659 return Legalized; 4660 } 4661 // for now, we use: { return popcount(~x & (x - 1)); } 4662 // unless the target has ctlz but not ctpop, in which case we use: 4663 // { return 32 - nlz(~x & (x-1)); } 4664 // Ref: "Hacker's Delight" by Henry Warren 4665 auto MIBCstNeg1 = MIRBuilder.buildConstant(Ty, -1); 4666 auto MIBNot = MIRBuilder.buildXor(Ty, SrcReg, MIBCstNeg1); 4667 auto MIBTmp = MIRBuilder.buildAnd( 4668 Ty, MIBNot, MIRBuilder.buildAdd(Ty, SrcReg, MIBCstNeg1)); 4669 if (!isSupported({TargetOpcode::G_CTPOP, {Ty, Ty}}) && 4670 isSupported({TargetOpcode::G_CTLZ, {Ty, Ty}})) { 4671 auto MIBCstLen = MIRBuilder.buildConstant(Ty, Len); 4672 MIRBuilder.buildSub(MI.getOperand(0), MIBCstLen, 4673 MIRBuilder.buildCTLZ(Ty, MIBTmp)); 4674 MI.eraseFromParent(); 4675 return Legalized; 4676 } 4677 MI.setDesc(TII.get(TargetOpcode::G_CTPOP)); 4678 MI.getOperand(1).setReg(MIBTmp.getReg(0)); 4679 return Legalized; 4680 } 4681 case TargetOpcode::G_CTPOP: { 4682 unsigned Size = Ty.getSizeInBits(); 4683 MachineIRBuilder &B = MIRBuilder; 4684 4685 // Count set bits in blocks of 2 bits. Default approach would be 4686 // B2Count = { val & 0x55555555 } + { (val >> 1) & 0x55555555 } 4687 // We use following formula instead: 4688 // B2Count = val - { (val >> 1) & 0x55555555 } 4689 // since it gives same result in blocks of 2 with one instruction less. 4690 auto C_1 = B.buildConstant(Ty, 1); 4691 auto B2Set1LoTo1Hi = B.buildLShr(Ty, MI.getOperand(1).getReg(), C_1); 4692 APInt B2Mask1HiTo0 = APInt::getSplat(Size, APInt(8, 0x55)); 4693 auto C_B2Mask1HiTo0 = B.buildConstant(Ty, B2Mask1HiTo0); 4694 auto B2Count1Hi = B.buildAnd(Ty, B2Set1LoTo1Hi, C_B2Mask1HiTo0); 4695 auto B2Count = B.buildSub(Ty, MI.getOperand(1).getReg(), B2Count1Hi); 4696 4697 // In order to get count in blocks of 4 add values from adjacent block of 2. 4698 // B4Count = { B2Count & 0x33333333 } + { (B2Count >> 2) & 0x33333333 } 4699 auto C_2 = B.buildConstant(Ty, 2); 4700 auto B4Set2LoTo2Hi = B.buildLShr(Ty, B2Count, C_2); 4701 APInt B4Mask2HiTo0 = APInt::getSplat(Size, APInt(8, 0x33)); 4702 auto C_B4Mask2HiTo0 = B.buildConstant(Ty, B4Mask2HiTo0); 4703 auto B4HiB2Count = B.buildAnd(Ty, B4Set2LoTo2Hi, C_B4Mask2HiTo0); 4704 auto B4LoB2Count = B.buildAnd(Ty, B2Count, C_B4Mask2HiTo0); 4705 auto B4Count = B.buildAdd(Ty, B4HiB2Count, B4LoB2Count); 4706 4707 // For count in blocks of 8 bits we don't have to mask high 4 bits before 4708 // addition since count value sits in range {0,...,8} and 4 bits are enough 4709 // to hold such binary values. After addition high 4 bits still hold count 4710 // of set bits in high 4 bit block, set them to zero and get 8 bit result. 4711 // B8Count = { B4Count + (B4Count >> 4) } & 0x0F0F0F0F 4712 auto C_4 = B.buildConstant(Ty, 4); 4713 auto B8HiB4Count = B.buildLShr(Ty, B4Count, C_4); 4714 auto B8CountDirty4Hi = B.buildAdd(Ty, B8HiB4Count, B4Count); 4715 APInt B8Mask4HiTo0 = APInt::getSplat(Size, APInt(8, 0x0F)); 4716 auto C_B8Mask4HiTo0 = B.buildConstant(Ty, B8Mask4HiTo0); 4717 auto B8Count = B.buildAnd(Ty, B8CountDirty4Hi, C_B8Mask4HiTo0); 4718 4719 assert(Size<=128 && "Scalar size is too large for CTPOP lower algorithm"); 4720 // 8 bits can hold CTPOP result of 128 bit int or smaller. Mul with this 4721 // bitmask will set 8 msb in ResTmp to sum of all B8Counts in 8 bit blocks. 4722 auto MulMask = B.buildConstant(Ty, APInt::getSplat(Size, APInt(8, 0x01))); 4723 auto ResTmp = B.buildMul(Ty, B8Count, MulMask); 4724 4725 // Shift count result from 8 high bits to low bits. 4726 auto C_SizeM8 = B.buildConstant(Ty, Size - 8); 4727 B.buildLShr(MI.getOperand(0).getReg(), ResTmp, C_SizeM8); 4728 4729 MI.eraseFromParent(); 4730 return Legalized; 4731 } 4732 } 4733 } 4734 4735 // Expand s32 = G_UITOFP s64 using bit operations to an IEEE float 4736 // representation. 4737 LegalizerHelper::LegalizeResult 4738 LegalizerHelper::lowerU64ToF32BitOps(MachineInstr &MI) { 4739 Register Dst = MI.getOperand(0).getReg(); 4740 Register Src = MI.getOperand(1).getReg(); 4741 const LLT S64 = LLT::scalar(64); 4742 const LLT S32 = LLT::scalar(32); 4743 const LLT S1 = LLT::scalar(1); 4744 4745 assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S32); 4746 4747 // unsigned cul2f(ulong u) { 4748 // uint lz = clz(u); 4749 // uint e = (u != 0) ? 127U + 63U - lz : 0; 4750 // u = (u << lz) & 0x7fffffffffffffffUL; 4751 // ulong t = u & 0xffffffffffUL; 4752 // uint v = (e << 23) | (uint)(u >> 40); 4753 // uint r = t > 0x8000000000UL ? 1U : (t == 0x8000000000UL ? v & 1U : 0U); 4754 // return as_float(v + r); 4755 // } 4756 4757 auto Zero32 = MIRBuilder.buildConstant(S32, 0); 4758 auto Zero64 = MIRBuilder.buildConstant(S64, 0); 4759 4760 auto LZ = MIRBuilder.buildCTLZ_ZERO_UNDEF(S32, Src); 4761 4762 auto K = MIRBuilder.buildConstant(S32, 127U + 63U); 4763 auto Sub = MIRBuilder.buildSub(S32, K, LZ); 4764 4765 auto NotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, Src, Zero64); 4766 auto E = MIRBuilder.buildSelect(S32, NotZero, Sub, Zero32); 4767 4768 auto Mask0 = MIRBuilder.buildConstant(S64, (-1ULL) >> 1); 4769 auto ShlLZ = MIRBuilder.buildShl(S64, Src, LZ); 4770 4771 auto U = MIRBuilder.buildAnd(S64, ShlLZ, Mask0); 4772 4773 auto Mask1 = MIRBuilder.buildConstant(S64, 0xffffffffffULL); 4774 auto T = MIRBuilder.buildAnd(S64, U, Mask1); 4775 4776 auto UShl = MIRBuilder.buildLShr(S64, U, MIRBuilder.buildConstant(S64, 40)); 4777 auto ShlE = MIRBuilder.buildShl(S32, E, MIRBuilder.buildConstant(S32, 23)); 4778 auto V = MIRBuilder.buildOr(S32, ShlE, MIRBuilder.buildTrunc(S32, UShl)); 4779 4780 auto C = MIRBuilder.buildConstant(S64, 0x8000000000ULL); 4781 auto RCmp = MIRBuilder.buildICmp(CmpInst::ICMP_UGT, S1, T, C); 4782 auto TCmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, T, C); 4783 auto One = MIRBuilder.buildConstant(S32, 1); 4784 4785 auto VTrunc1 = MIRBuilder.buildAnd(S32, V, One); 4786 auto Select0 = MIRBuilder.buildSelect(S32, TCmp, VTrunc1, Zero32); 4787 auto R = MIRBuilder.buildSelect(S32, RCmp, One, Select0); 4788 MIRBuilder.buildAdd(Dst, V, R); 4789 4790 MI.eraseFromParent(); 4791 return Legalized; 4792 } 4793 4794 LegalizerHelper::LegalizeResult 4795 LegalizerHelper::lowerUITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { 4796 Register Dst = MI.getOperand(0).getReg(); 4797 Register Src = MI.getOperand(1).getReg(); 4798 LLT DstTy = MRI.getType(Dst); 4799 LLT SrcTy = MRI.getType(Src); 4800 4801 if (SrcTy == LLT::scalar(1)) { 4802 auto True = MIRBuilder.buildFConstant(DstTy, 1.0); 4803 auto False = MIRBuilder.buildFConstant(DstTy, 0.0); 4804 MIRBuilder.buildSelect(Dst, Src, True, False); 4805 MI.eraseFromParent(); 4806 return Legalized; 4807 } 4808 4809 if (SrcTy != LLT::scalar(64)) 4810 return UnableToLegalize; 4811 4812 if (DstTy == LLT::scalar(32)) { 4813 // TODO: SelectionDAG has several alternative expansions to port which may 4814 // be more reasonble depending on the available instructions. If a target 4815 // has sitofp, does not have CTLZ, or can efficiently use f64 as an 4816 // intermediate type, this is probably worse. 4817 return lowerU64ToF32BitOps(MI); 4818 } 4819 4820 return UnableToLegalize; 4821 } 4822 4823 LegalizerHelper::LegalizeResult 4824 LegalizerHelper::lowerSITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { 4825 Register Dst = MI.getOperand(0).getReg(); 4826 Register Src = MI.getOperand(1).getReg(); 4827 LLT DstTy = MRI.getType(Dst); 4828 LLT SrcTy = MRI.getType(Src); 4829 4830 const LLT S64 = LLT::scalar(64); 4831 const LLT S32 = LLT::scalar(32); 4832 const LLT S1 = LLT::scalar(1); 4833 4834 if (SrcTy == S1) { 4835 auto True = MIRBuilder.buildFConstant(DstTy, -1.0); 4836 auto False = MIRBuilder.buildFConstant(DstTy, 0.0); 4837 MIRBuilder.buildSelect(Dst, Src, True, False); 4838 MI.eraseFromParent(); 4839 return Legalized; 4840 } 4841 4842 if (SrcTy != S64) 4843 return UnableToLegalize; 4844 4845 if (DstTy == S32) { 4846 // signed cl2f(long l) { 4847 // long s = l >> 63; 4848 // float r = cul2f((l + s) ^ s); 4849 // return s ? -r : r; 4850 // } 4851 Register L = Src; 4852 auto SignBit = MIRBuilder.buildConstant(S64, 63); 4853 auto S = MIRBuilder.buildAShr(S64, L, SignBit); 4854 4855 auto LPlusS = MIRBuilder.buildAdd(S64, L, S); 4856 auto Xor = MIRBuilder.buildXor(S64, LPlusS, S); 4857 auto R = MIRBuilder.buildUITOFP(S32, Xor); 4858 4859 auto RNeg = MIRBuilder.buildFNeg(S32, R); 4860 auto SignNotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, S, 4861 MIRBuilder.buildConstant(S64, 0)); 4862 MIRBuilder.buildSelect(Dst, SignNotZero, RNeg, R); 4863 MI.eraseFromParent(); 4864 return Legalized; 4865 } 4866 4867 return UnableToLegalize; 4868 } 4869 4870 LegalizerHelper::LegalizeResult 4871 LegalizerHelper::lowerFPTOUI(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { 4872 Register Dst = MI.getOperand(0).getReg(); 4873 Register Src = MI.getOperand(1).getReg(); 4874 LLT DstTy = MRI.getType(Dst); 4875 LLT SrcTy = MRI.getType(Src); 4876 const LLT S64 = LLT::scalar(64); 4877 const LLT S32 = LLT::scalar(32); 4878 4879 if (SrcTy != S64 && SrcTy != S32) 4880 return UnableToLegalize; 4881 if (DstTy != S32 && DstTy != S64) 4882 return UnableToLegalize; 4883 4884 // FPTOSI gives same result as FPTOUI for positive signed integers. 4885 // FPTOUI needs to deal with fp values that convert to unsigned integers 4886 // greater or equal to 2^31 for float or 2^63 for double. For brevity 2^Exp. 4887 4888 APInt TwoPExpInt = APInt::getSignMask(DstTy.getSizeInBits()); 4889 APFloat TwoPExpFP(SrcTy.getSizeInBits() == 32 ? APFloat::IEEEsingle() 4890 : APFloat::IEEEdouble(), 4891 APInt::getNullValue(SrcTy.getSizeInBits())); 4892 TwoPExpFP.convertFromAPInt(TwoPExpInt, false, APFloat::rmNearestTiesToEven); 4893 4894 MachineInstrBuilder FPTOSI = MIRBuilder.buildFPTOSI(DstTy, Src); 4895 4896 MachineInstrBuilder Threshold = MIRBuilder.buildFConstant(SrcTy, TwoPExpFP); 4897 // For fp Value greater or equal to Threshold(2^Exp), we use FPTOSI on 4898 // (Value - 2^Exp) and add 2^Exp by setting highest bit in result to 1. 4899 MachineInstrBuilder FSub = MIRBuilder.buildFSub(SrcTy, Src, Threshold); 4900 MachineInstrBuilder ResLowBits = MIRBuilder.buildFPTOSI(DstTy, FSub); 4901 MachineInstrBuilder ResHighBit = MIRBuilder.buildConstant(DstTy, TwoPExpInt); 4902 MachineInstrBuilder Res = MIRBuilder.buildXor(DstTy, ResLowBits, ResHighBit); 4903 4904 const LLT S1 = LLT::scalar(1); 4905 4906 MachineInstrBuilder FCMP = 4907 MIRBuilder.buildFCmp(CmpInst::FCMP_ULT, S1, Src, Threshold); 4908 MIRBuilder.buildSelect(Dst, FCMP, FPTOSI, Res); 4909 4910 MI.eraseFromParent(); 4911 return Legalized; 4912 } 4913 4914 LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPTOSI(MachineInstr &MI) { 4915 Register Dst = MI.getOperand(0).getReg(); 4916 Register Src = MI.getOperand(1).getReg(); 4917 LLT DstTy = MRI.getType(Dst); 4918 LLT SrcTy = MRI.getType(Src); 4919 const LLT S64 = LLT::scalar(64); 4920 const LLT S32 = LLT::scalar(32); 4921 4922 // FIXME: Only f32 to i64 conversions are supported. 4923 if (SrcTy.getScalarType() != S32 || DstTy.getScalarType() != S64) 4924 return UnableToLegalize; 4925 4926 // Expand f32 -> i64 conversion 4927 // This algorithm comes from compiler-rt's implementation of fixsfdi: 4928 // https://github.com/llvm/llvm-project/blob/master/compiler-rt/lib/builtins/fixsfdi.c 4929 4930 unsigned SrcEltBits = SrcTy.getScalarSizeInBits(); 4931 4932 auto ExponentMask = MIRBuilder.buildConstant(SrcTy, 0x7F800000); 4933 auto ExponentLoBit = MIRBuilder.buildConstant(SrcTy, 23); 4934 4935 auto AndExpMask = MIRBuilder.buildAnd(SrcTy, Src, ExponentMask); 4936 auto ExponentBits = MIRBuilder.buildLShr(SrcTy, AndExpMask, ExponentLoBit); 4937 4938 auto SignMask = MIRBuilder.buildConstant(SrcTy, 4939 APInt::getSignMask(SrcEltBits)); 4940 auto AndSignMask = MIRBuilder.buildAnd(SrcTy, Src, SignMask); 4941 auto SignLowBit = MIRBuilder.buildConstant(SrcTy, SrcEltBits - 1); 4942 auto Sign = MIRBuilder.buildAShr(SrcTy, AndSignMask, SignLowBit); 4943 Sign = MIRBuilder.buildSExt(DstTy, Sign); 4944 4945 auto MantissaMask = MIRBuilder.buildConstant(SrcTy, 0x007FFFFF); 4946 auto AndMantissaMask = MIRBuilder.buildAnd(SrcTy, Src, MantissaMask); 4947 auto K = MIRBuilder.buildConstant(SrcTy, 0x00800000); 4948 4949 auto R = MIRBuilder.buildOr(SrcTy, AndMantissaMask, K); 4950 R = MIRBuilder.buildZExt(DstTy, R); 4951 4952 auto Bias = MIRBuilder.buildConstant(SrcTy, 127); 4953 auto Exponent = MIRBuilder.buildSub(SrcTy, ExponentBits, Bias); 4954 auto SubExponent = MIRBuilder.buildSub(SrcTy, Exponent, ExponentLoBit); 4955 auto ExponentSub = MIRBuilder.buildSub(SrcTy, ExponentLoBit, Exponent); 4956 4957 auto Shl = MIRBuilder.buildShl(DstTy, R, SubExponent); 4958 auto Srl = MIRBuilder.buildLShr(DstTy, R, ExponentSub); 4959 4960 const LLT S1 = LLT::scalar(1); 4961 auto CmpGt = MIRBuilder.buildICmp(CmpInst::ICMP_SGT, 4962 S1, Exponent, ExponentLoBit); 4963 4964 R = MIRBuilder.buildSelect(DstTy, CmpGt, Shl, Srl); 4965 4966 auto XorSign = MIRBuilder.buildXor(DstTy, R, Sign); 4967 auto Ret = MIRBuilder.buildSub(DstTy, XorSign, Sign); 4968 4969 auto ZeroSrcTy = MIRBuilder.buildConstant(SrcTy, 0); 4970 4971 auto ExponentLt0 = MIRBuilder.buildICmp(CmpInst::ICMP_SLT, 4972 S1, Exponent, ZeroSrcTy); 4973 4974 auto ZeroDstTy = MIRBuilder.buildConstant(DstTy, 0); 4975 MIRBuilder.buildSelect(Dst, ExponentLt0, ZeroDstTy, Ret); 4976 4977 MI.eraseFromParent(); 4978 return Legalized; 4979 } 4980 4981 // f64 -> f16 conversion using round-to-nearest-even rounding mode. 4982 LegalizerHelper::LegalizeResult 4983 LegalizerHelper::lowerFPTRUNC_F64_TO_F16(MachineInstr &MI) { 4984 Register Dst = MI.getOperand(0).getReg(); 4985 Register Src = MI.getOperand(1).getReg(); 4986 4987 if (MRI.getType(Src).isVector()) // TODO: Handle vectors directly. 4988 return UnableToLegalize; 4989 4990 const unsigned ExpMask = 0x7ff; 4991 const unsigned ExpBiasf64 = 1023; 4992 const unsigned ExpBiasf16 = 15; 4993 const LLT S32 = LLT::scalar(32); 4994 const LLT S1 = LLT::scalar(1); 4995 4996 auto Unmerge = MIRBuilder.buildUnmerge(S32, Src); 4997 Register U = Unmerge.getReg(0); 4998 Register UH = Unmerge.getReg(1); 4999 5000 auto E = MIRBuilder.buildLShr(S32, UH, MIRBuilder.buildConstant(S32, 20)); 5001 E = MIRBuilder.buildAnd(S32, E, MIRBuilder.buildConstant(S32, ExpMask)); 5002 5003 // Subtract the fp64 exponent bias (1023) to get the real exponent and 5004 // add the f16 bias (15) to get the biased exponent for the f16 format. 5005 E = MIRBuilder.buildAdd( 5006 S32, E, MIRBuilder.buildConstant(S32, -ExpBiasf64 + ExpBiasf16)); 5007 5008 auto M = MIRBuilder.buildLShr(S32, UH, MIRBuilder.buildConstant(S32, 8)); 5009 M = MIRBuilder.buildAnd(S32, M, MIRBuilder.buildConstant(S32, 0xffe)); 5010 5011 auto MaskedSig = MIRBuilder.buildAnd(S32, UH, 5012 MIRBuilder.buildConstant(S32, 0x1ff)); 5013 MaskedSig = MIRBuilder.buildOr(S32, MaskedSig, U); 5014 5015 auto Zero = MIRBuilder.buildConstant(S32, 0); 5016 auto SigCmpNE0 = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, MaskedSig, Zero); 5017 auto Lo40Set = MIRBuilder.buildZExt(S32, SigCmpNE0); 5018 M = MIRBuilder.buildOr(S32, M, Lo40Set); 5019 5020 // (M != 0 ? 0x0200 : 0) | 0x7c00; 5021 auto Bits0x200 = MIRBuilder.buildConstant(S32, 0x0200); 5022 auto CmpM_NE0 = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, M, Zero); 5023 auto SelectCC = MIRBuilder.buildSelect(S32, CmpM_NE0, Bits0x200, Zero); 5024 5025 auto Bits0x7c00 = MIRBuilder.buildConstant(S32, 0x7c00); 5026 auto I = MIRBuilder.buildOr(S32, SelectCC, Bits0x7c00); 5027 5028 // N = M | (E << 12); 5029 auto EShl12 = MIRBuilder.buildShl(S32, E, MIRBuilder.buildConstant(S32, 12)); 5030 auto N = MIRBuilder.buildOr(S32, M, EShl12); 5031 5032 // B = clamp(1-E, 0, 13); 5033 auto One = MIRBuilder.buildConstant(S32, 1); 5034 auto OneSubExp = MIRBuilder.buildSub(S32, One, E); 5035 auto B = MIRBuilder.buildSMax(S32, OneSubExp, Zero); 5036 B = MIRBuilder.buildSMin(S32, B, MIRBuilder.buildConstant(S32, 13)); 5037 5038 auto SigSetHigh = MIRBuilder.buildOr(S32, M, 5039 MIRBuilder.buildConstant(S32, 0x1000)); 5040 5041 auto D = MIRBuilder.buildLShr(S32, SigSetHigh, B); 5042 auto D0 = MIRBuilder.buildShl(S32, D, B); 5043 5044 auto D0_NE_SigSetHigh = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, 5045 D0, SigSetHigh); 5046 auto D1 = MIRBuilder.buildZExt(S32, D0_NE_SigSetHigh); 5047 D = MIRBuilder.buildOr(S32, D, D1); 5048 5049 auto CmpELtOne = MIRBuilder.buildICmp(CmpInst::ICMP_SLT, S1, E, One); 5050 auto V = MIRBuilder.buildSelect(S32, CmpELtOne, D, N); 5051 5052 auto VLow3 = MIRBuilder.buildAnd(S32, V, MIRBuilder.buildConstant(S32, 7)); 5053 V = MIRBuilder.buildLShr(S32, V, MIRBuilder.buildConstant(S32, 2)); 5054 5055 auto VLow3Eq3 = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, VLow3, 5056 MIRBuilder.buildConstant(S32, 3)); 5057 auto V0 = MIRBuilder.buildZExt(S32, VLow3Eq3); 5058 5059 auto VLow3Gt5 = MIRBuilder.buildICmp(CmpInst::ICMP_SGT, S1, VLow3, 5060 MIRBuilder.buildConstant(S32, 5)); 5061 auto V1 = MIRBuilder.buildZExt(S32, VLow3Gt5); 5062 5063 V1 = MIRBuilder.buildOr(S32, V0, V1); 5064 V = MIRBuilder.buildAdd(S32, V, V1); 5065 5066 auto CmpEGt30 = MIRBuilder.buildICmp(CmpInst::ICMP_SGT, S1, 5067 E, MIRBuilder.buildConstant(S32, 30)); 5068 V = MIRBuilder.buildSelect(S32, CmpEGt30, 5069 MIRBuilder.buildConstant(S32, 0x7c00), V); 5070 5071 auto CmpEGt1039 = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, 5072 E, MIRBuilder.buildConstant(S32, 1039)); 5073 V = MIRBuilder.buildSelect(S32, CmpEGt1039, I, V); 5074 5075 // Extract the sign bit. 5076 auto Sign = MIRBuilder.buildLShr(S32, UH, MIRBuilder.buildConstant(S32, 16)); 5077 Sign = MIRBuilder.buildAnd(S32, Sign, MIRBuilder.buildConstant(S32, 0x8000)); 5078 5079 // Insert the sign bit 5080 V = MIRBuilder.buildOr(S32, Sign, V); 5081 5082 MIRBuilder.buildTrunc(Dst, V); 5083 MI.eraseFromParent(); 5084 return Legalized; 5085 } 5086 5087 LegalizerHelper::LegalizeResult 5088 LegalizerHelper::lowerFPTRUNC(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { 5089 Register Dst = MI.getOperand(0).getReg(); 5090 Register Src = MI.getOperand(1).getReg(); 5091 5092 LLT DstTy = MRI.getType(Dst); 5093 LLT SrcTy = MRI.getType(Src); 5094 const LLT S64 = LLT::scalar(64); 5095 const LLT S16 = LLT::scalar(16); 5096 5097 if (DstTy.getScalarType() == S16 && SrcTy.getScalarType() == S64) 5098 return lowerFPTRUNC_F64_TO_F16(MI); 5099 5100 return UnableToLegalize; 5101 } 5102 5103 // TODO: If RHS is a constant SelectionDAGBuilder expands this into a 5104 // multiplication tree. 5105 LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPOWI(MachineInstr &MI) { 5106 Register Dst = MI.getOperand(0).getReg(); 5107 Register Src0 = MI.getOperand(1).getReg(); 5108 Register Src1 = MI.getOperand(2).getReg(); 5109 LLT Ty = MRI.getType(Dst); 5110 5111 auto CvtSrc1 = MIRBuilder.buildSITOFP(Ty, Src1); 5112 MIRBuilder.buildFPow(Dst, Src0, CvtSrc1, MI.getFlags()); 5113 MI.eraseFromParent(); 5114 return Legalized; 5115 } 5116 5117 static CmpInst::Predicate minMaxToCompare(unsigned Opc) { 5118 switch (Opc) { 5119 case TargetOpcode::G_SMIN: 5120 return CmpInst::ICMP_SLT; 5121 case TargetOpcode::G_SMAX: 5122 return CmpInst::ICMP_SGT; 5123 case TargetOpcode::G_UMIN: 5124 return CmpInst::ICMP_ULT; 5125 case TargetOpcode::G_UMAX: 5126 return CmpInst::ICMP_UGT; 5127 default: 5128 llvm_unreachable("not in integer min/max"); 5129 } 5130 } 5131 5132 LegalizerHelper::LegalizeResult 5133 LegalizerHelper::lowerMinMax(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { 5134 Register Dst = MI.getOperand(0).getReg(); 5135 Register Src0 = MI.getOperand(1).getReg(); 5136 Register Src1 = MI.getOperand(2).getReg(); 5137 5138 const CmpInst::Predicate Pred = minMaxToCompare(MI.getOpcode()); 5139 LLT CmpType = MRI.getType(Dst).changeElementSize(1); 5140 5141 auto Cmp = MIRBuilder.buildICmp(Pred, CmpType, Src0, Src1); 5142 MIRBuilder.buildSelect(Dst, Cmp, Src0, Src1); 5143 5144 MI.eraseFromParent(); 5145 return Legalized; 5146 } 5147 5148 LegalizerHelper::LegalizeResult 5149 LegalizerHelper::lowerFCopySign(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { 5150 Register Dst = MI.getOperand(0).getReg(); 5151 Register Src0 = MI.getOperand(1).getReg(); 5152 Register Src1 = MI.getOperand(2).getReg(); 5153 5154 const LLT Src0Ty = MRI.getType(Src0); 5155 const LLT Src1Ty = MRI.getType(Src1); 5156 5157 const int Src0Size = Src0Ty.getScalarSizeInBits(); 5158 const int Src1Size = Src1Ty.getScalarSizeInBits(); 5159 5160 auto SignBitMask = MIRBuilder.buildConstant( 5161 Src0Ty, APInt::getSignMask(Src0Size)); 5162 5163 auto NotSignBitMask = MIRBuilder.buildConstant( 5164 Src0Ty, APInt::getLowBitsSet(Src0Size, Src0Size - 1)); 5165 5166 auto And0 = MIRBuilder.buildAnd(Src0Ty, Src0, NotSignBitMask); 5167 MachineInstr *Or; 5168 5169 if (Src0Ty == Src1Ty) { 5170 auto And1 = MIRBuilder.buildAnd(Src1Ty, Src1, SignBitMask); 5171 Or = MIRBuilder.buildOr(Dst, And0, And1); 5172 } else if (Src0Size > Src1Size) { 5173 auto ShiftAmt = MIRBuilder.buildConstant(Src0Ty, Src0Size - Src1Size); 5174 auto Zext = MIRBuilder.buildZExt(Src0Ty, Src1); 5175 auto Shift = MIRBuilder.buildShl(Src0Ty, Zext, ShiftAmt); 5176 auto And1 = MIRBuilder.buildAnd(Src0Ty, Shift, SignBitMask); 5177 Or = MIRBuilder.buildOr(Dst, And0, And1); 5178 } else { 5179 auto ShiftAmt = MIRBuilder.buildConstant(Src1Ty, Src1Size - Src0Size); 5180 auto Shift = MIRBuilder.buildLShr(Src1Ty, Src1, ShiftAmt); 5181 auto Trunc = MIRBuilder.buildTrunc(Src0Ty, Shift); 5182 auto And1 = MIRBuilder.buildAnd(Src0Ty, Trunc, SignBitMask); 5183 Or = MIRBuilder.buildOr(Dst, And0, And1); 5184 } 5185 5186 // Be careful about setting nsz/nnan/ninf on every instruction, since the 5187 // constants are a nan and -0.0, but the final result should preserve 5188 // everything. 5189 if (unsigned Flags = MI.getFlags()) 5190 Or->setFlags(Flags); 5191 5192 MI.eraseFromParent(); 5193 return Legalized; 5194 } 5195 5196 LegalizerHelper::LegalizeResult 5197 LegalizerHelper::lowerFMinNumMaxNum(MachineInstr &MI) { 5198 unsigned NewOp = MI.getOpcode() == TargetOpcode::G_FMINNUM ? 5199 TargetOpcode::G_FMINNUM_IEEE : TargetOpcode::G_FMAXNUM_IEEE; 5200 5201 Register Dst = MI.getOperand(0).getReg(); 5202 Register Src0 = MI.getOperand(1).getReg(); 5203 Register Src1 = MI.getOperand(2).getReg(); 5204 LLT Ty = MRI.getType(Dst); 5205 5206 if (!MI.getFlag(MachineInstr::FmNoNans)) { 5207 // Insert canonicalizes if it's possible we need to quiet to get correct 5208 // sNaN behavior. 5209 5210 // Note this must be done here, and not as an optimization combine in the 5211 // absence of a dedicate quiet-snan instruction as we're using an 5212 // omni-purpose G_FCANONICALIZE. 5213 if (!isKnownNeverSNaN(Src0, MRI)) 5214 Src0 = MIRBuilder.buildFCanonicalize(Ty, Src0, MI.getFlags()).getReg(0); 5215 5216 if (!isKnownNeverSNaN(Src1, MRI)) 5217 Src1 = MIRBuilder.buildFCanonicalize(Ty, Src1, MI.getFlags()).getReg(0); 5218 } 5219 5220 // If there are no nans, it's safe to simply replace this with the non-IEEE 5221 // version. 5222 MIRBuilder.buildInstr(NewOp, {Dst}, {Src0, Src1}, MI.getFlags()); 5223 MI.eraseFromParent(); 5224 return Legalized; 5225 } 5226 5227 LegalizerHelper::LegalizeResult LegalizerHelper::lowerFMad(MachineInstr &MI) { 5228 // Expand G_FMAD a, b, c -> G_FADD (G_FMUL a, b), c 5229 Register DstReg = MI.getOperand(0).getReg(); 5230 LLT Ty = MRI.getType(DstReg); 5231 unsigned Flags = MI.getFlags(); 5232 5233 auto Mul = MIRBuilder.buildFMul(Ty, MI.getOperand(1), MI.getOperand(2), 5234 Flags); 5235 MIRBuilder.buildFAdd(DstReg, Mul, MI.getOperand(3), Flags); 5236 MI.eraseFromParent(); 5237 return Legalized; 5238 } 5239 5240 LegalizerHelper::LegalizeResult 5241 LegalizerHelper::lowerIntrinsicRound(MachineInstr &MI) { 5242 Register DstReg = MI.getOperand(0).getReg(); 5243 Register X = MI.getOperand(1).getReg(); 5244 const unsigned Flags = MI.getFlags(); 5245 const LLT Ty = MRI.getType(DstReg); 5246 const LLT CondTy = Ty.changeElementSize(1); 5247 5248 // round(x) => 5249 // t = trunc(x); 5250 // d = fabs(x - t); 5251 // o = copysign(1.0f, x); 5252 // return t + (d >= 0.5 ? o : 0.0); 5253 5254 auto T = MIRBuilder.buildIntrinsicTrunc(Ty, X, Flags); 5255 5256 auto Diff = MIRBuilder.buildFSub(Ty, X, T, Flags); 5257 auto AbsDiff = MIRBuilder.buildFAbs(Ty, Diff, Flags); 5258 auto Zero = MIRBuilder.buildFConstant(Ty, 0.0); 5259 auto One = MIRBuilder.buildFConstant(Ty, 1.0); 5260 auto Half = MIRBuilder.buildFConstant(Ty, 0.5); 5261 auto SignOne = MIRBuilder.buildFCopysign(Ty, One, X); 5262 5263 auto Cmp = MIRBuilder.buildFCmp(CmpInst::FCMP_OGE, CondTy, AbsDiff, Half, 5264 Flags); 5265 auto Sel = MIRBuilder.buildSelect(Ty, Cmp, SignOne, Zero, Flags); 5266 5267 MIRBuilder.buildFAdd(DstReg, T, Sel, Flags); 5268 5269 MI.eraseFromParent(); 5270 return Legalized; 5271 } 5272 5273 LegalizerHelper::LegalizeResult 5274 LegalizerHelper::lowerFFloor(MachineInstr &MI) { 5275 Register DstReg = MI.getOperand(0).getReg(); 5276 Register SrcReg = MI.getOperand(1).getReg(); 5277 unsigned Flags = MI.getFlags(); 5278 LLT Ty = MRI.getType(DstReg); 5279 const LLT CondTy = Ty.changeElementSize(1); 5280 5281 // result = trunc(src); 5282 // if (src < 0.0 && src != result) 5283 // result += -1.0. 5284 5285 auto Trunc = MIRBuilder.buildIntrinsicTrunc(Ty, SrcReg, Flags); 5286 auto Zero = MIRBuilder.buildFConstant(Ty, 0.0); 5287 5288 auto Lt0 = MIRBuilder.buildFCmp(CmpInst::FCMP_OLT, CondTy, 5289 SrcReg, Zero, Flags); 5290 auto NeTrunc = MIRBuilder.buildFCmp(CmpInst::FCMP_ONE, CondTy, 5291 SrcReg, Trunc, Flags); 5292 auto And = MIRBuilder.buildAnd(CondTy, Lt0, NeTrunc); 5293 auto AddVal = MIRBuilder.buildSITOFP(Ty, And); 5294 5295 MIRBuilder.buildFAdd(DstReg, Trunc, AddVal, Flags); 5296 MI.eraseFromParent(); 5297 return Legalized; 5298 } 5299 5300 LegalizerHelper::LegalizeResult 5301 LegalizerHelper::lowerMergeValues(MachineInstr &MI) { 5302 const unsigned NumOps = MI.getNumOperands(); 5303 Register DstReg = MI.getOperand(0).getReg(); 5304 Register Src0Reg = MI.getOperand(1).getReg(); 5305 LLT DstTy = MRI.getType(DstReg); 5306 LLT SrcTy = MRI.getType(Src0Reg); 5307 unsigned PartSize = SrcTy.getSizeInBits(); 5308 5309 LLT WideTy = LLT::scalar(DstTy.getSizeInBits()); 5310 Register ResultReg = MIRBuilder.buildZExt(WideTy, Src0Reg).getReg(0); 5311 5312 for (unsigned I = 2; I != NumOps; ++I) { 5313 const unsigned Offset = (I - 1) * PartSize; 5314 5315 Register SrcReg = MI.getOperand(I).getReg(); 5316 auto ZextInput = MIRBuilder.buildZExt(WideTy, SrcReg); 5317 5318 Register NextResult = I + 1 == NumOps && WideTy == DstTy ? DstReg : 5319 MRI.createGenericVirtualRegister(WideTy); 5320 5321 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, Offset); 5322 auto Shl = MIRBuilder.buildShl(WideTy, ZextInput, ShiftAmt); 5323 MIRBuilder.buildOr(NextResult, ResultReg, Shl); 5324 ResultReg = NextResult; 5325 } 5326 5327 if (DstTy.isPointer()) { 5328 if (MIRBuilder.getDataLayout().isNonIntegralAddressSpace( 5329 DstTy.getAddressSpace())) { 5330 LLVM_DEBUG(dbgs() << "Not casting nonintegral address space\n"); 5331 return UnableToLegalize; 5332 } 5333 5334 MIRBuilder.buildIntToPtr(DstReg, ResultReg); 5335 } 5336 5337 MI.eraseFromParent(); 5338 return Legalized; 5339 } 5340 5341 LegalizerHelper::LegalizeResult 5342 LegalizerHelper::lowerUnmergeValues(MachineInstr &MI) { 5343 const unsigned NumDst = MI.getNumOperands() - 1; 5344 Register SrcReg = MI.getOperand(NumDst).getReg(); 5345 Register Dst0Reg = MI.getOperand(0).getReg(); 5346 LLT DstTy = MRI.getType(Dst0Reg); 5347 if (DstTy.isPointer()) 5348 return UnableToLegalize; // TODO 5349 5350 SrcReg = coerceToScalar(SrcReg); 5351 if (!SrcReg) 5352 return UnableToLegalize; 5353 5354 // Expand scalarizing unmerge as bitcast to integer and shift. 5355 LLT IntTy = MRI.getType(SrcReg); 5356 5357 MIRBuilder.buildTrunc(Dst0Reg, SrcReg); 5358 5359 const unsigned DstSize = DstTy.getSizeInBits(); 5360 unsigned Offset = DstSize; 5361 for (unsigned I = 1; I != NumDst; ++I, Offset += DstSize) { 5362 auto ShiftAmt = MIRBuilder.buildConstant(IntTy, Offset); 5363 auto Shift = MIRBuilder.buildLShr(IntTy, SrcReg, ShiftAmt); 5364 MIRBuilder.buildTrunc(MI.getOperand(I), Shift); 5365 } 5366 5367 MI.eraseFromParent(); 5368 return Legalized; 5369 } 5370 5371 /// Lower a vector extract by writing the vector to a stack temporary and 5372 /// reloading the element. 5373 /// 5374 /// %dst = G_EXTRACT_VECTOR_ELT %vec, %idx 5375 /// => 5376 /// %stack_temp = G_FRAME_INDEX 5377 /// G_STORE %vec, %stack_temp 5378 /// %idx = clamp(%idx, %vec.getNumElements()) 5379 /// %element_ptr = G_PTR_ADD %stack_temp, %idx 5380 /// %dst = G_LOAD %element_ptr 5381 LegalizerHelper::LegalizeResult 5382 LegalizerHelper::lowerExtractVectorElt(MachineInstr &MI) { 5383 Register DstReg = MI.getOperand(0).getReg(); 5384 Register SrcVec = MI.getOperand(1).getReg(); 5385 Register Idx = MI.getOperand(2).getReg(); 5386 LLT VecTy = MRI.getType(SrcVec); 5387 LLT EltTy = VecTy.getElementType(); 5388 if (!EltTy.isByteSized()) { // Not implemented. 5389 LLVM_DEBUG(dbgs() << "Can't handle non-byte element vectors yet\n"); 5390 return UnableToLegalize; 5391 } 5392 5393 unsigned EltBytes = EltTy.getSizeInBytes(); 5394 Align StoreAlign = getStackTemporaryAlignment(VecTy); 5395 Align LoadAlign; 5396 5397 MachinePointerInfo PtrInfo; 5398 auto StackTemp = createStackTemporary(TypeSize::Fixed(VecTy.getSizeInBytes()), 5399 StoreAlign, PtrInfo); 5400 MIRBuilder.buildStore(SrcVec, StackTemp, PtrInfo, StoreAlign); 5401 5402 // Get the pointer to the element, and be sure not to hit undefined behavior 5403 // if the index is out of bounds. 5404 Register LoadPtr = getVectorElementPointer(StackTemp.getReg(0), VecTy, Idx); 5405 5406 int64_t IdxVal; 5407 if (mi_match(Idx, MRI, m_ICst(IdxVal))) { 5408 int64_t Offset = IdxVal * EltBytes; 5409 PtrInfo = PtrInfo.getWithOffset(Offset); 5410 LoadAlign = commonAlignment(StoreAlign, Offset); 5411 } else { 5412 // We lose information with a variable offset. 5413 LoadAlign = getStackTemporaryAlignment(EltTy); 5414 PtrInfo = MachinePointerInfo(MRI.getType(LoadPtr).getAddressSpace()); 5415 } 5416 5417 MIRBuilder.buildLoad(DstReg, LoadPtr, PtrInfo, LoadAlign); 5418 MI.eraseFromParent(); 5419 return Legalized; 5420 } 5421 5422 LegalizerHelper::LegalizeResult 5423 LegalizerHelper::lowerShuffleVector(MachineInstr &MI) { 5424 Register DstReg = MI.getOperand(0).getReg(); 5425 Register Src0Reg = MI.getOperand(1).getReg(); 5426 Register Src1Reg = MI.getOperand(2).getReg(); 5427 LLT Src0Ty = MRI.getType(Src0Reg); 5428 LLT DstTy = MRI.getType(DstReg); 5429 LLT IdxTy = LLT::scalar(32); 5430 5431 ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask(); 5432 5433 if (DstTy.isScalar()) { 5434 if (Src0Ty.isVector()) 5435 return UnableToLegalize; 5436 5437 // This is just a SELECT. 5438 assert(Mask.size() == 1 && "Expected a single mask element"); 5439 Register Val; 5440 if (Mask[0] < 0 || Mask[0] > 1) 5441 Val = MIRBuilder.buildUndef(DstTy).getReg(0); 5442 else 5443 Val = Mask[0] == 0 ? Src0Reg : Src1Reg; 5444 MIRBuilder.buildCopy(DstReg, Val); 5445 MI.eraseFromParent(); 5446 return Legalized; 5447 } 5448 5449 Register Undef; 5450 SmallVector<Register, 32> BuildVec; 5451 LLT EltTy = DstTy.getElementType(); 5452 5453 for (int Idx : Mask) { 5454 if (Idx < 0) { 5455 if (!Undef.isValid()) 5456 Undef = MIRBuilder.buildUndef(EltTy).getReg(0); 5457 BuildVec.push_back(Undef); 5458 continue; 5459 } 5460 5461 if (Src0Ty.isScalar()) { 5462 BuildVec.push_back(Idx == 0 ? Src0Reg : Src1Reg); 5463 } else { 5464 int NumElts = Src0Ty.getNumElements(); 5465 Register SrcVec = Idx < NumElts ? Src0Reg : Src1Reg; 5466 int ExtractIdx = Idx < NumElts ? Idx : Idx - NumElts; 5467 auto IdxK = MIRBuilder.buildConstant(IdxTy, ExtractIdx); 5468 auto Extract = MIRBuilder.buildExtractVectorElement(EltTy, SrcVec, IdxK); 5469 BuildVec.push_back(Extract.getReg(0)); 5470 } 5471 } 5472 5473 MIRBuilder.buildBuildVector(DstReg, BuildVec); 5474 MI.eraseFromParent(); 5475 return Legalized; 5476 } 5477 5478 LegalizerHelper::LegalizeResult 5479 LegalizerHelper::lowerDynStackAlloc(MachineInstr &MI) { 5480 const auto &MF = *MI.getMF(); 5481 const auto &TFI = *MF.getSubtarget().getFrameLowering(); 5482 if (TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp) 5483 return UnableToLegalize; 5484 5485 Register Dst = MI.getOperand(0).getReg(); 5486 Register AllocSize = MI.getOperand(1).getReg(); 5487 Align Alignment = assumeAligned(MI.getOperand(2).getImm()); 5488 5489 LLT PtrTy = MRI.getType(Dst); 5490 LLT IntPtrTy = LLT::scalar(PtrTy.getSizeInBits()); 5491 5492 const auto &TLI = *MF.getSubtarget().getTargetLowering(); 5493 Register SPReg = TLI.getStackPointerRegisterToSaveRestore(); 5494 auto SPTmp = MIRBuilder.buildCopy(PtrTy, SPReg); 5495 SPTmp = MIRBuilder.buildCast(IntPtrTy, SPTmp); 5496 5497 // Subtract the final alloc from the SP. We use G_PTRTOINT here so we don't 5498 // have to generate an extra instruction to negate the alloc and then use 5499 // G_PTR_ADD to add the negative offset. 5500 auto Alloc = MIRBuilder.buildSub(IntPtrTy, SPTmp, AllocSize); 5501 if (Alignment > Align(1)) { 5502 APInt AlignMask(IntPtrTy.getSizeInBits(), Alignment.value(), true); 5503 AlignMask.negate(); 5504 auto AlignCst = MIRBuilder.buildConstant(IntPtrTy, AlignMask); 5505 Alloc = MIRBuilder.buildAnd(IntPtrTy, Alloc, AlignCst); 5506 } 5507 5508 SPTmp = MIRBuilder.buildCast(PtrTy, Alloc); 5509 MIRBuilder.buildCopy(SPReg, SPTmp); 5510 MIRBuilder.buildCopy(Dst, SPTmp); 5511 5512 MI.eraseFromParent(); 5513 return Legalized; 5514 } 5515 5516 LegalizerHelper::LegalizeResult 5517 LegalizerHelper::lowerExtract(MachineInstr &MI) { 5518 Register Dst = MI.getOperand(0).getReg(); 5519 Register Src = MI.getOperand(1).getReg(); 5520 unsigned Offset = MI.getOperand(2).getImm(); 5521 5522 LLT DstTy = MRI.getType(Dst); 5523 LLT SrcTy = MRI.getType(Src); 5524 5525 if (DstTy.isScalar() && 5526 (SrcTy.isScalar() || 5527 (SrcTy.isVector() && DstTy == SrcTy.getElementType()))) { 5528 LLT SrcIntTy = SrcTy; 5529 if (!SrcTy.isScalar()) { 5530 SrcIntTy = LLT::scalar(SrcTy.getSizeInBits()); 5531 Src = MIRBuilder.buildBitcast(SrcIntTy, Src).getReg(0); 5532 } 5533 5534 if (Offset == 0) 5535 MIRBuilder.buildTrunc(Dst, Src); 5536 else { 5537 auto ShiftAmt = MIRBuilder.buildConstant(SrcIntTy, Offset); 5538 auto Shr = MIRBuilder.buildLShr(SrcIntTy, Src, ShiftAmt); 5539 MIRBuilder.buildTrunc(Dst, Shr); 5540 } 5541 5542 MI.eraseFromParent(); 5543 return Legalized; 5544 } 5545 5546 return UnableToLegalize; 5547 } 5548 5549 LegalizerHelper::LegalizeResult LegalizerHelper::lowerInsert(MachineInstr &MI) { 5550 Register Dst = MI.getOperand(0).getReg(); 5551 Register Src = MI.getOperand(1).getReg(); 5552 Register InsertSrc = MI.getOperand(2).getReg(); 5553 uint64_t Offset = MI.getOperand(3).getImm(); 5554 5555 LLT DstTy = MRI.getType(Src); 5556 LLT InsertTy = MRI.getType(InsertSrc); 5557 5558 if (InsertTy.isVector() || 5559 (DstTy.isVector() && DstTy.getElementType() != InsertTy)) 5560 return UnableToLegalize; 5561 5562 const DataLayout &DL = MIRBuilder.getDataLayout(); 5563 if ((DstTy.isPointer() && 5564 DL.isNonIntegralAddressSpace(DstTy.getAddressSpace())) || 5565 (InsertTy.isPointer() && 5566 DL.isNonIntegralAddressSpace(InsertTy.getAddressSpace()))) { 5567 LLVM_DEBUG(dbgs() << "Not casting non-integral address space integer\n"); 5568 return UnableToLegalize; 5569 } 5570 5571 LLT IntDstTy = DstTy; 5572 5573 if (!DstTy.isScalar()) { 5574 IntDstTy = LLT::scalar(DstTy.getSizeInBits()); 5575 Src = MIRBuilder.buildCast(IntDstTy, Src).getReg(0); 5576 } 5577 5578 if (!InsertTy.isScalar()) { 5579 const LLT IntInsertTy = LLT::scalar(InsertTy.getSizeInBits()); 5580 InsertSrc = MIRBuilder.buildPtrToInt(IntInsertTy, InsertSrc).getReg(0); 5581 } 5582 5583 Register ExtInsSrc = MIRBuilder.buildZExt(IntDstTy, InsertSrc).getReg(0); 5584 if (Offset != 0) { 5585 auto ShiftAmt = MIRBuilder.buildConstant(IntDstTy, Offset); 5586 ExtInsSrc = MIRBuilder.buildShl(IntDstTy, ExtInsSrc, ShiftAmt).getReg(0); 5587 } 5588 5589 APInt MaskVal = APInt::getBitsSetWithWrap( 5590 DstTy.getSizeInBits(), Offset + InsertTy.getSizeInBits(), Offset); 5591 5592 auto Mask = MIRBuilder.buildConstant(IntDstTy, MaskVal); 5593 auto MaskedSrc = MIRBuilder.buildAnd(IntDstTy, Src, Mask); 5594 auto Or = MIRBuilder.buildOr(IntDstTy, MaskedSrc, ExtInsSrc); 5595 5596 MIRBuilder.buildCast(Dst, Or); 5597 MI.eraseFromParent(); 5598 return Legalized; 5599 } 5600 5601 LegalizerHelper::LegalizeResult 5602 LegalizerHelper::lowerSADDO_SSUBO(MachineInstr &MI) { 5603 Register Dst0 = MI.getOperand(0).getReg(); 5604 Register Dst1 = MI.getOperand(1).getReg(); 5605 Register LHS = MI.getOperand(2).getReg(); 5606 Register RHS = MI.getOperand(3).getReg(); 5607 const bool IsAdd = MI.getOpcode() == TargetOpcode::G_SADDO; 5608 5609 LLT Ty = MRI.getType(Dst0); 5610 LLT BoolTy = MRI.getType(Dst1); 5611 5612 if (IsAdd) 5613 MIRBuilder.buildAdd(Dst0, LHS, RHS); 5614 else 5615 MIRBuilder.buildSub(Dst0, LHS, RHS); 5616 5617 // TODO: If SADDSAT/SSUBSAT is legal, compare results to detect overflow. 5618 5619 auto Zero = MIRBuilder.buildConstant(Ty, 0); 5620 5621 // For an addition, the result should be less than one of the operands (LHS) 5622 // if and only if the other operand (RHS) is negative, otherwise there will 5623 // be overflow. 5624 // For a subtraction, the result should be less than one of the operands 5625 // (LHS) if and only if the other operand (RHS) is (non-zero) positive, 5626 // otherwise there will be overflow. 5627 auto ResultLowerThanLHS = 5628 MIRBuilder.buildICmp(CmpInst::ICMP_SLT, BoolTy, Dst0, LHS); 5629 auto ConditionRHS = MIRBuilder.buildICmp( 5630 IsAdd ? CmpInst::ICMP_SLT : CmpInst::ICMP_SGT, BoolTy, RHS, Zero); 5631 5632 MIRBuilder.buildXor(Dst1, ConditionRHS, ResultLowerThanLHS); 5633 MI.eraseFromParent(); 5634 return Legalized; 5635 } 5636 5637 LegalizerHelper::LegalizeResult 5638 LegalizerHelper::lowerAddSubSatToMinMax(MachineInstr &MI) { 5639 Register Res = MI.getOperand(0).getReg(); 5640 Register LHS = MI.getOperand(1).getReg(); 5641 Register RHS = MI.getOperand(2).getReg(); 5642 LLT Ty = MRI.getType(Res); 5643 bool IsSigned; 5644 bool IsAdd; 5645 unsigned BaseOp; 5646 switch (MI.getOpcode()) { 5647 default: 5648 llvm_unreachable("unexpected addsat/subsat opcode"); 5649 case TargetOpcode::G_UADDSAT: 5650 IsSigned = false; 5651 IsAdd = true; 5652 BaseOp = TargetOpcode::G_ADD; 5653 break; 5654 case TargetOpcode::G_SADDSAT: 5655 IsSigned = true; 5656 IsAdd = true; 5657 BaseOp = TargetOpcode::G_ADD; 5658 break; 5659 case TargetOpcode::G_USUBSAT: 5660 IsSigned = false; 5661 IsAdd = false; 5662 BaseOp = TargetOpcode::G_SUB; 5663 break; 5664 case TargetOpcode::G_SSUBSAT: 5665 IsSigned = true; 5666 IsAdd = false; 5667 BaseOp = TargetOpcode::G_SUB; 5668 break; 5669 } 5670 5671 if (IsSigned) { 5672 // sadd.sat(a, b) -> 5673 // hi = 0x7fffffff - smax(a, 0) 5674 // lo = 0x80000000 - smin(a, 0) 5675 // a + smin(smax(lo, b), hi) 5676 // ssub.sat(a, b) -> 5677 // lo = smax(a, -1) - 0x7fffffff 5678 // hi = smin(a, -1) - 0x80000000 5679 // a - smin(smax(lo, b), hi) 5680 // TODO: AMDGPU can use a "median of 3" instruction here: 5681 // a +/- med3(lo, b, hi) 5682 uint64_t NumBits = Ty.getScalarSizeInBits(); 5683 auto MaxVal = 5684 MIRBuilder.buildConstant(Ty, APInt::getSignedMaxValue(NumBits)); 5685 auto MinVal = 5686 MIRBuilder.buildConstant(Ty, APInt::getSignedMinValue(NumBits)); 5687 MachineInstrBuilder Hi, Lo; 5688 if (IsAdd) { 5689 auto Zero = MIRBuilder.buildConstant(Ty, 0); 5690 Hi = MIRBuilder.buildSub(Ty, MaxVal, MIRBuilder.buildSMax(Ty, LHS, Zero)); 5691 Lo = MIRBuilder.buildSub(Ty, MinVal, MIRBuilder.buildSMin(Ty, LHS, Zero)); 5692 } else { 5693 auto NegOne = MIRBuilder.buildConstant(Ty, -1); 5694 Lo = MIRBuilder.buildSub(Ty, MIRBuilder.buildSMax(Ty, LHS, NegOne), 5695 MaxVal); 5696 Hi = MIRBuilder.buildSub(Ty, MIRBuilder.buildSMin(Ty, LHS, NegOne), 5697 MinVal); 5698 } 5699 auto RHSClamped = 5700 MIRBuilder.buildSMin(Ty, MIRBuilder.buildSMax(Ty, Lo, RHS), Hi); 5701 MIRBuilder.buildInstr(BaseOp, {Res}, {LHS, RHSClamped}); 5702 } else { 5703 // uadd.sat(a, b) -> a + umin(~a, b) 5704 // usub.sat(a, b) -> a - umin(a, b) 5705 Register Not = IsAdd ? MIRBuilder.buildNot(Ty, LHS).getReg(0) : LHS; 5706 auto Min = MIRBuilder.buildUMin(Ty, Not, RHS); 5707 MIRBuilder.buildInstr(BaseOp, {Res}, {LHS, Min}); 5708 } 5709 5710 MI.eraseFromParent(); 5711 return Legalized; 5712 } 5713 5714 LegalizerHelper::LegalizeResult 5715 LegalizerHelper::lowerAddSubSatToAddoSubo(MachineInstr &MI) { 5716 Register Res = MI.getOperand(0).getReg(); 5717 Register LHS = MI.getOperand(1).getReg(); 5718 Register RHS = MI.getOperand(2).getReg(); 5719 LLT Ty = MRI.getType(Res); 5720 LLT BoolTy = Ty.changeElementSize(1); 5721 bool IsSigned; 5722 bool IsAdd; 5723 unsigned OverflowOp; 5724 switch (MI.getOpcode()) { 5725 default: 5726 llvm_unreachable("unexpected addsat/subsat opcode"); 5727 case TargetOpcode::G_UADDSAT: 5728 IsSigned = false; 5729 IsAdd = true; 5730 OverflowOp = TargetOpcode::G_UADDO; 5731 break; 5732 case TargetOpcode::G_SADDSAT: 5733 IsSigned = true; 5734 IsAdd = true; 5735 OverflowOp = TargetOpcode::G_SADDO; 5736 break; 5737 case TargetOpcode::G_USUBSAT: 5738 IsSigned = false; 5739 IsAdd = false; 5740 OverflowOp = TargetOpcode::G_USUBO; 5741 break; 5742 case TargetOpcode::G_SSUBSAT: 5743 IsSigned = true; 5744 IsAdd = false; 5745 OverflowOp = TargetOpcode::G_SSUBO; 5746 break; 5747 } 5748 5749 auto OverflowRes = 5750 MIRBuilder.buildInstr(OverflowOp, {Ty, BoolTy}, {LHS, RHS}); 5751 Register Tmp = OverflowRes.getReg(0); 5752 Register Ov = OverflowRes.getReg(1); 5753 MachineInstrBuilder Clamp; 5754 if (IsSigned) { 5755 // sadd.sat(a, b) -> 5756 // {tmp, ov} = saddo(a, b) 5757 // ov ? (tmp >>s 31) + 0x80000000 : r 5758 // ssub.sat(a, b) -> 5759 // {tmp, ov} = ssubo(a, b) 5760 // ov ? (tmp >>s 31) + 0x80000000 : r 5761 uint64_t NumBits = Ty.getScalarSizeInBits(); 5762 auto ShiftAmount = MIRBuilder.buildConstant(Ty, NumBits - 1); 5763 auto Sign = MIRBuilder.buildAShr(Ty, Tmp, ShiftAmount); 5764 auto MinVal = 5765 MIRBuilder.buildConstant(Ty, APInt::getSignedMinValue(NumBits)); 5766 Clamp = MIRBuilder.buildAdd(Ty, Sign, MinVal); 5767 } else { 5768 // uadd.sat(a, b) -> 5769 // {tmp, ov} = uaddo(a, b) 5770 // ov ? 0xffffffff : tmp 5771 // usub.sat(a, b) -> 5772 // {tmp, ov} = usubo(a, b) 5773 // ov ? 0 : tmp 5774 Clamp = MIRBuilder.buildConstant(Ty, IsAdd ? -1 : 0); 5775 } 5776 MIRBuilder.buildSelect(Res, Ov, Clamp, Tmp); 5777 5778 MI.eraseFromParent(); 5779 return Legalized; 5780 } 5781 5782 LegalizerHelper::LegalizeResult 5783 LegalizerHelper::lowerBswap(MachineInstr &MI) { 5784 Register Dst = MI.getOperand(0).getReg(); 5785 Register Src = MI.getOperand(1).getReg(); 5786 const LLT Ty = MRI.getType(Src); 5787 unsigned SizeInBytes = (Ty.getScalarSizeInBits() + 7) / 8; 5788 unsigned BaseShiftAmt = (SizeInBytes - 1) * 8; 5789 5790 // Swap most and least significant byte, set remaining bytes in Res to zero. 5791 auto ShiftAmt = MIRBuilder.buildConstant(Ty, BaseShiftAmt); 5792 auto LSByteShiftedLeft = MIRBuilder.buildShl(Ty, Src, ShiftAmt); 5793 auto MSByteShiftedRight = MIRBuilder.buildLShr(Ty, Src, ShiftAmt); 5794 auto Res = MIRBuilder.buildOr(Ty, MSByteShiftedRight, LSByteShiftedLeft); 5795 5796 // Set i-th high/low byte in Res to i-th low/high byte from Src. 5797 for (unsigned i = 1; i < SizeInBytes / 2; ++i) { 5798 // AND with Mask leaves byte i unchanged and sets remaining bytes to 0. 5799 APInt APMask(SizeInBytes * 8, 0xFF << (i * 8)); 5800 auto Mask = MIRBuilder.buildConstant(Ty, APMask); 5801 auto ShiftAmt = MIRBuilder.buildConstant(Ty, BaseShiftAmt - 16 * i); 5802 // Low byte shifted left to place of high byte: (Src & Mask) << ShiftAmt. 5803 auto LoByte = MIRBuilder.buildAnd(Ty, Src, Mask); 5804 auto LoShiftedLeft = MIRBuilder.buildShl(Ty, LoByte, ShiftAmt); 5805 Res = MIRBuilder.buildOr(Ty, Res, LoShiftedLeft); 5806 // High byte shifted right to place of low byte: (Src >> ShiftAmt) & Mask. 5807 auto SrcShiftedRight = MIRBuilder.buildLShr(Ty, Src, ShiftAmt); 5808 auto HiShiftedRight = MIRBuilder.buildAnd(Ty, SrcShiftedRight, Mask); 5809 Res = MIRBuilder.buildOr(Ty, Res, HiShiftedRight); 5810 } 5811 Res.getInstr()->getOperand(0).setReg(Dst); 5812 5813 MI.eraseFromParent(); 5814 return Legalized; 5815 } 5816 5817 //{ (Src & Mask) >> N } | { (Src << N) & Mask } 5818 static MachineInstrBuilder SwapN(unsigned N, DstOp Dst, MachineIRBuilder &B, 5819 MachineInstrBuilder Src, APInt Mask) { 5820 const LLT Ty = Dst.getLLTTy(*B.getMRI()); 5821 MachineInstrBuilder C_N = B.buildConstant(Ty, N); 5822 MachineInstrBuilder MaskLoNTo0 = B.buildConstant(Ty, Mask); 5823 auto LHS = B.buildLShr(Ty, B.buildAnd(Ty, Src, MaskLoNTo0), C_N); 5824 auto RHS = B.buildAnd(Ty, B.buildShl(Ty, Src, C_N), MaskLoNTo0); 5825 return B.buildOr(Dst, LHS, RHS); 5826 } 5827 5828 LegalizerHelper::LegalizeResult 5829 LegalizerHelper::lowerBitreverse(MachineInstr &MI) { 5830 Register Dst = MI.getOperand(0).getReg(); 5831 Register Src = MI.getOperand(1).getReg(); 5832 const LLT Ty = MRI.getType(Src); 5833 unsigned Size = Ty.getSizeInBits(); 5834 5835 MachineInstrBuilder BSWAP = 5836 MIRBuilder.buildInstr(TargetOpcode::G_BSWAP, {Ty}, {Src}); 5837 5838 // swap high and low 4 bits in 8 bit blocks 7654|3210 -> 3210|7654 5839 // [(val & 0xF0F0F0F0) >> 4] | [(val & 0x0F0F0F0F) << 4] 5840 // -> [(val & 0xF0F0F0F0) >> 4] | [(val << 4) & 0xF0F0F0F0] 5841 MachineInstrBuilder Swap4 = 5842 SwapN(4, Ty, MIRBuilder, BSWAP, APInt::getSplat(Size, APInt(8, 0xF0))); 5843 5844 // swap high and low 2 bits in 4 bit blocks 32|10 76|54 -> 10|32 54|76 5845 // [(val & 0xCCCCCCCC) >> 2] & [(val & 0x33333333) << 2] 5846 // -> [(val & 0xCCCCCCCC) >> 2] & [(val << 2) & 0xCCCCCCCC] 5847 MachineInstrBuilder Swap2 = 5848 SwapN(2, Ty, MIRBuilder, Swap4, APInt::getSplat(Size, APInt(8, 0xCC))); 5849 5850 // swap high and low 1 bit in 2 bit blocks 1|0 3|2 5|4 7|6 -> 0|1 2|3 4|5 6|7 5851 // [(val & 0xAAAAAAAA) >> 1] & [(val & 0x55555555) << 1] 5852 // -> [(val & 0xAAAAAAAA) >> 1] & [(val << 1) & 0xAAAAAAAA] 5853 SwapN(1, Dst, MIRBuilder, Swap2, APInt::getSplat(Size, APInt(8, 0xAA))); 5854 5855 MI.eraseFromParent(); 5856 return Legalized; 5857 } 5858 5859 LegalizerHelper::LegalizeResult 5860 LegalizerHelper::lowerReadWriteRegister(MachineInstr &MI) { 5861 MachineFunction &MF = MIRBuilder.getMF(); 5862 const TargetSubtargetInfo &STI = MF.getSubtarget(); 5863 const TargetLowering *TLI = STI.getTargetLowering(); 5864 5865 bool IsRead = MI.getOpcode() == TargetOpcode::G_READ_REGISTER; 5866 int NameOpIdx = IsRead ? 1 : 0; 5867 int ValRegIndex = IsRead ? 0 : 1; 5868 5869 Register ValReg = MI.getOperand(ValRegIndex).getReg(); 5870 const LLT Ty = MRI.getType(ValReg); 5871 const MDString *RegStr = cast<MDString>( 5872 cast<MDNode>(MI.getOperand(NameOpIdx).getMetadata())->getOperand(0)); 5873 5874 Register PhysReg = TLI->getRegisterByName(RegStr->getString().data(), Ty, MF); 5875 if (!PhysReg.isValid()) 5876 return UnableToLegalize; 5877 5878 if (IsRead) 5879 MIRBuilder.buildCopy(ValReg, PhysReg); 5880 else 5881 MIRBuilder.buildCopy(PhysReg, ValReg); 5882 5883 MI.eraseFromParent(); 5884 return Legalized; 5885 } 5886