1 //===- X86InstructionSelector.cpp ----------------------------*- C++ -*-==// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 /// \file 10 /// This file implements the targeting of the InstructionSelector class for 11 /// X86. 12 /// \todo This should be generated by TableGen. 13 //===----------------------------------------------------------------------===// 14 15 #include "X86InstrBuilder.h" 16 #include "X86InstrInfo.h" 17 #include "X86RegisterBankInfo.h" 18 #include "X86RegisterInfo.h" 19 #include "X86Subtarget.h" 20 #include "X86TargetMachine.h" 21 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 22 #include "llvm/CodeGen/GlobalISel/Utils.h" 23 #include "llvm/CodeGen/MachineBasicBlock.h" 24 #include "llvm/CodeGen/MachineFunction.h" 25 #include "llvm/CodeGen/MachineInstr.h" 26 #include "llvm/CodeGen/MachineInstrBuilder.h" 27 #include "llvm/CodeGen/MachineOperand.h" 28 #include "llvm/CodeGen/MachineRegisterInfo.h" 29 #include "llvm/IR/Type.h" 30 #include "llvm/Support/Debug.h" 31 #include "llvm/Support/raw_ostream.h" 32 33 #define DEBUG_TYPE "X86-isel" 34 35 using namespace llvm; 36 37 #ifndef LLVM_BUILD_GLOBAL_ISEL 38 #error "You shouldn't build this" 39 #endif 40 41 namespace { 42 43 #define GET_GLOBALISEL_PREDICATE_BITSET 44 #include "X86GenGlobalISel.inc" 45 #undef GET_GLOBALISEL_PREDICATE_BITSET 46 47 class X86InstructionSelector : public InstructionSelector { 48 public: 49 X86InstructionSelector(const X86TargetMachine &TM, const X86Subtarget &STI, 50 const X86RegisterBankInfo &RBI); 51 52 bool select(MachineInstr &I) const override; 53 54 private: 55 /// tblgen-erated 'select' implementation, used as the initial selector for 56 /// the patterns that don't require complex C++. 57 bool selectImpl(MachineInstr &I) const; 58 59 // TODO: remove after suported by Tablegen-erated instruction selection. 60 unsigned getLoadStoreOp(LLT &Ty, const RegisterBank &RB, unsigned Opc, 61 uint64_t Alignment) const; 62 63 bool selectLoadStoreOp(MachineInstr &I, MachineRegisterInfo &MRI, 64 MachineFunction &MF) const; 65 bool selectFrameIndexOrGep(MachineInstr &I, MachineRegisterInfo &MRI, 66 MachineFunction &MF) const; 67 bool selectConstant(MachineInstr &I, MachineRegisterInfo &MRI, 68 MachineFunction &MF) const; 69 bool selectTrunc(MachineInstr &I, MachineRegisterInfo &MRI, 70 MachineFunction &MF) const; 71 bool selectZext(MachineInstr &I, MachineRegisterInfo &MRI, 72 MachineFunction &MF) const; 73 bool selectCmp(MachineInstr &I, MachineRegisterInfo &MRI, 74 MachineFunction &MF) const; 75 bool selectUadde(MachineInstr &I, MachineRegisterInfo &MRI, 76 MachineFunction &MF) const; 77 bool selectCopy(MachineInstr &I, MachineRegisterInfo &MRI) const; 78 bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI, 79 MachineFunction &MF) const; 80 bool selectInsert(MachineInstr &I, MachineRegisterInfo &MRI, 81 MachineFunction &MF) const; 82 bool selectExtract(MachineInstr &I, MachineRegisterInfo &MRI, 83 MachineFunction &MF) const; 84 85 // emit insert subreg instruction and insert it before MachineInstr &I 86 bool emitInsertSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I, 87 MachineRegisterInfo &MRI, MachineFunction &MF) const; 88 // emit extract subreg instruction and insert it before MachineInstr &I 89 bool emitExtractSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I, 90 MachineRegisterInfo &MRI, MachineFunction &MF) const; 91 92 const TargetRegisterClass *getRegClass(LLT Ty, const RegisterBank &RB) const; 93 const TargetRegisterClass *getRegClass(LLT Ty, unsigned Reg, 94 MachineRegisterInfo &MRI) const; 95 96 const X86TargetMachine &TM; 97 const X86Subtarget &STI; 98 const X86InstrInfo &TII; 99 const X86RegisterInfo &TRI; 100 const X86RegisterBankInfo &RBI; 101 102 #define GET_GLOBALISEL_PREDICATES_DECL 103 #include "X86GenGlobalISel.inc" 104 #undef GET_GLOBALISEL_PREDICATES_DECL 105 106 #define GET_GLOBALISEL_TEMPORARIES_DECL 107 #include "X86GenGlobalISel.inc" 108 #undef GET_GLOBALISEL_TEMPORARIES_DECL 109 }; 110 111 } // end anonymous namespace 112 113 #define GET_GLOBALISEL_IMPL 114 #include "X86GenGlobalISel.inc" 115 #undef GET_GLOBALISEL_IMPL 116 117 X86InstructionSelector::X86InstructionSelector(const X86TargetMachine &TM, 118 const X86Subtarget &STI, 119 const X86RegisterBankInfo &RBI) 120 : InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()), 121 TRI(*STI.getRegisterInfo()), RBI(RBI), 122 #define GET_GLOBALISEL_PREDICATES_INIT 123 #include "X86GenGlobalISel.inc" 124 #undef GET_GLOBALISEL_PREDICATES_INIT 125 #define GET_GLOBALISEL_TEMPORARIES_INIT 126 #include "X86GenGlobalISel.inc" 127 #undef GET_GLOBALISEL_TEMPORARIES_INIT 128 { 129 } 130 131 // FIXME: This should be target-independent, inferred from the types declared 132 // for each class in the bank. 133 const TargetRegisterClass * 134 X86InstructionSelector::getRegClass(LLT Ty, const RegisterBank &RB) const { 135 if (RB.getID() == X86::GPRRegBankID) { 136 if (Ty.getSizeInBits() <= 8) 137 return &X86::GR8RegClass; 138 if (Ty.getSizeInBits() == 16) 139 return &X86::GR16RegClass; 140 if (Ty.getSizeInBits() == 32) 141 return &X86::GR32RegClass; 142 if (Ty.getSizeInBits() == 64) 143 return &X86::GR64RegClass; 144 } 145 if (RB.getID() == X86::VECRRegBankID) { 146 if (Ty.getSizeInBits() == 32) 147 return STI.hasAVX512() ? &X86::FR32XRegClass : &X86::FR32RegClass; 148 if (Ty.getSizeInBits() == 64) 149 return STI.hasAVX512() ? &X86::FR64XRegClass : &X86::FR64RegClass; 150 if (Ty.getSizeInBits() == 128) 151 return STI.hasAVX512() ? &X86::VR128XRegClass : &X86::VR128RegClass; 152 if (Ty.getSizeInBits() == 256) 153 return STI.hasAVX512() ? &X86::VR256XRegClass : &X86::VR256RegClass; 154 if (Ty.getSizeInBits() == 512) 155 return &X86::VR512RegClass; 156 } 157 158 llvm_unreachable("Unknown RegBank!"); 159 } 160 161 const TargetRegisterClass * 162 X86InstructionSelector::getRegClass(LLT Ty, unsigned Reg, 163 MachineRegisterInfo &MRI) const { 164 const RegisterBank &RegBank = *RBI.getRegBank(Reg, MRI, TRI); 165 return getRegClass(Ty, RegBank); 166 } 167 168 // Set X86 Opcode and constrain DestReg. 169 bool X86InstructionSelector::selectCopy(MachineInstr &I, 170 MachineRegisterInfo &MRI) const { 171 172 unsigned DstReg = I.getOperand(0).getReg(); 173 if (TargetRegisterInfo::isPhysicalRegister(DstReg)) { 174 assert(I.isCopy() && "Generic operators do not allow physical registers"); 175 return true; 176 } 177 178 const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI); 179 const unsigned DstSize = MRI.getType(DstReg).getSizeInBits(); 180 unsigned SrcReg = I.getOperand(1).getReg(); 181 const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI); 182 183 assert((!TargetRegisterInfo::isPhysicalRegister(SrcReg) || I.isCopy()) && 184 "No phys reg on generic operators"); 185 assert((DstSize == SrcSize || 186 // Copies are a mean to setup initial types, the number of 187 // bits may not exactly match. 188 (TargetRegisterInfo::isPhysicalRegister(SrcReg) && 189 DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI))) && 190 "Copy with different width?!"); 191 192 const TargetRegisterClass *RC = nullptr; 193 194 switch (RegBank.getID()) { 195 case X86::GPRRegBankID: 196 assert((DstSize <= 64) && "GPRs cannot get more than 64-bit width values."); 197 RC = getRegClass(MRI.getType(DstReg), RegBank); 198 199 // Change the physical register 200 if (SrcSize > DstSize && TargetRegisterInfo::isPhysicalRegister(SrcReg)) { 201 if (RC == &X86::GR32RegClass) 202 I.getOperand(1).setSubReg(X86::sub_32bit); 203 else if (RC == &X86::GR16RegClass) 204 I.getOperand(1).setSubReg(X86::sub_16bit); 205 else if (RC == &X86::GR8RegClass) 206 I.getOperand(1).setSubReg(X86::sub_8bit); 207 208 I.getOperand(1).substPhysReg(SrcReg, TRI); 209 } 210 break; 211 case X86::VECRRegBankID: 212 RC = getRegClass(MRI.getType(DstReg), RegBank); 213 break; 214 default: 215 llvm_unreachable("Unknown RegBank!"); 216 } 217 218 // No need to constrain SrcReg. It will get constrained when 219 // we hit another of its use or its defs. 220 // Copies do not have constraints. 221 const TargetRegisterClass *OldRC = MRI.getRegClassOrNull(DstReg); 222 if (!OldRC || !RC->hasSubClassEq(OldRC)) { 223 if (!RBI.constrainGenericRegister(DstReg, *RC, MRI)) { 224 DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode()) 225 << " operand\n"); 226 return false; 227 } 228 } 229 I.setDesc(TII.get(X86::COPY)); 230 return true; 231 } 232 233 bool X86InstructionSelector::select(MachineInstr &I) const { 234 assert(I.getParent() && "Instruction should be in a basic block!"); 235 assert(I.getParent()->getParent() && "Instruction should be in a function!"); 236 237 MachineBasicBlock &MBB = *I.getParent(); 238 MachineFunction &MF = *MBB.getParent(); 239 MachineRegisterInfo &MRI = MF.getRegInfo(); 240 241 unsigned Opcode = I.getOpcode(); 242 if (!isPreISelGenericOpcode(Opcode)) { 243 // Certain non-generic instructions also need some special handling. 244 245 if (I.isCopy()) 246 return selectCopy(I, MRI); 247 248 // TODO: handle more cases - LOAD_STACK_GUARD, PHI 249 return true; 250 } 251 252 assert(I.getNumOperands() == I.getNumExplicitOperands() && 253 "Generic instruction has unexpected implicit operands\n"); 254 255 if (selectImpl(I)) 256 return true; 257 258 DEBUG(dbgs() << " C++ instruction selection: "; I.print(dbgs())); 259 260 // TODO: This should be implemented by tblgen. 261 if (selectLoadStoreOp(I, MRI, MF)) 262 return true; 263 if (selectFrameIndexOrGep(I, MRI, MF)) 264 return true; 265 if (selectConstant(I, MRI, MF)) 266 return true; 267 if (selectTrunc(I, MRI, MF)) 268 return true; 269 if (selectZext(I, MRI, MF)) 270 return true; 271 if (selectCmp(I, MRI, MF)) 272 return true; 273 if (selectUadde(I, MRI, MF)) 274 return true; 275 if (selectMergeValues(I, MRI, MF)) 276 return true; 277 if (selectExtract(I, MRI, MF)) 278 return true; 279 if (selectInsert(I, MRI, MF)) 280 return true; 281 282 return false; 283 } 284 285 unsigned X86InstructionSelector::getLoadStoreOp(LLT &Ty, const RegisterBank &RB, 286 unsigned Opc, 287 uint64_t Alignment) const { 288 bool Isload = (Opc == TargetOpcode::G_LOAD); 289 bool HasAVX = STI.hasAVX(); 290 bool HasAVX512 = STI.hasAVX512(); 291 bool HasVLX = STI.hasVLX(); 292 293 if (Ty == LLT::scalar(8)) { 294 if (X86::GPRRegBankID == RB.getID()) 295 return Isload ? X86::MOV8rm : X86::MOV8mr; 296 } else if (Ty == LLT::scalar(16)) { 297 if (X86::GPRRegBankID == RB.getID()) 298 return Isload ? X86::MOV16rm : X86::MOV16mr; 299 } else if (Ty == LLT::scalar(32) || Ty == LLT::pointer(0, 32)) { 300 if (X86::GPRRegBankID == RB.getID()) 301 return Isload ? X86::MOV32rm : X86::MOV32mr; 302 if (X86::VECRRegBankID == RB.getID()) 303 return Isload ? (HasAVX512 ? X86::VMOVSSZrm 304 : HasAVX ? X86::VMOVSSrm : X86::MOVSSrm) 305 : (HasAVX512 ? X86::VMOVSSZmr 306 : HasAVX ? X86::VMOVSSmr : X86::MOVSSmr); 307 } else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64)) { 308 if (X86::GPRRegBankID == RB.getID()) 309 return Isload ? X86::MOV64rm : X86::MOV64mr; 310 if (X86::VECRRegBankID == RB.getID()) 311 return Isload ? (HasAVX512 ? X86::VMOVSDZrm 312 : HasAVX ? X86::VMOVSDrm : X86::MOVSDrm) 313 : (HasAVX512 ? X86::VMOVSDZmr 314 : HasAVX ? X86::VMOVSDmr : X86::MOVSDmr); 315 } else if (Ty.isVector() && Ty.getSizeInBits() == 128) { 316 if (Alignment >= 16) 317 return Isload ? (HasVLX ? X86::VMOVAPSZ128rm 318 : HasAVX512 319 ? X86::VMOVAPSZ128rm_NOVLX 320 : HasAVX ? X86::VMOVAPSrm : X86::MOVAPSrm) 321 : (HasVLX ? X86::VMOVAPSZ128mr 322 : HasAVX512 323 ? X86::VMOVAPSZ128mr_NOVLX 324 : HasAVX ? X86::VMOVAPSmr : X86::MOVAPSmr); 325 else 326 return Isload ? (HasVLX ? X86::VMOVUPSZ128rm 327 : HasAVX512 328 ? X86::VMOVUPSZ128rm_NOVLX 329 : HasAVX ? X86::VMOVUPSrm : X86::MOVUPSrm) 330 : (HasVLX ? X86::VMOVUPSZ128mr 331 : HasAVX512 332 ? X86::VMOVUPSZ128mr_NOVLX 333 : HasAVX ? X86::VMOVUPSmr : X86::MOVUPSmr); 334 } else if (Ty.isVector() && Ty.getSizeInBits() == 256) { 335 if (Alignment >= 32) 336 return Isload ? (HasVLX ? X86::VMOVAPSZ256rm 337 : HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX 338 : X86::VMOVAPSYrm) 339 : (HasVLX ? X86::VMOVAPSZ256mr 340 : HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX 341 : X86::VMOVAPSYmr); 342 else 343 return Isload ? (HasVLX ? X86::VMOVUPSZ256rm 344 : HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX 345 : X86::VMOVUPSYrm) 346 : (HasVLX ? X86::VMOVUPSZ256mr 347 : HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX 348 : X86::VMOVUPSYmr); 349 } else if (Ty.isVector() && Ty.getSizeInBits() == 512) { 350 if (Alignment >= 64) 351 return Isload ? X86::VMOVAPSZrm : X86::VMOVAPSZmr; 352 else 353 return Isload ? X86::VMOVUPSZrm : X86::VMOVUPSZmr; 354 } 355 return Opc; 356 } 357 358 // Fill in an address from the given instruction. 359 void X86SelectAddress(const MachineInstr &I, const MachineRegisterInfo &MRI, 360 X86AddressMode &AM) { 361 362 assert(I.getOperand(0).isReg() && "unsupported opperand."); 363 assert(MRI.getType(I.getOperand(0).getReg()).isPointer() && 364 "unsupported type."); 365 366 if (I.getOpcode() == TargetOpcode::G_GEP) { 367 if (auto COff = getConstantVRegVal(I.getOperand(2).getReg(), MRI)) { 368 int64_t Imm = *COff; 369 if (isInt<32>(Imm)) { // Check for displacement overflow. 370 AM.Disp = static_cast<int32_t>(Imm); 371 AM.Base.Reg = I.getOperand(1).getReg(); 372 return; 373 } 374 } 375 } else if (I.getOpcode() == TargetOpcode::G_FRAME_INDEX) { 376 AM.Base.FrameIndex = I.getOperand(1).getIndex(); 377 AM.BaseType = X86AddressMode::FrameIndexBase; 378 return; 379 } 380 381 // Default behavior. 382 AM.Base.Reg = I.getOperand(0).getReg(); 383 return; 384 } 385 386 bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I, 387 MachineRegisterInfo &MRI, 388 MachineFunction &MF) const { 389 390 unsigned Opc = I.getOpcode(); 391 392 if (Opc != TargetOpcode::G_STORE && Opc != TargetOpcode::G_LOAD) 393 return false; 394 395 const unsigned DefReg = I.getOperand(0).getReg(); 396 LLT Ty = MRI.getType(DefReg); 397 const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI); 398 399 auto &MemOp = **I.memoperands_begin(); 400 if (MemOp.getOrdering() != AtomicOrdering::NotAtomic) { 401 DEBUG(dbgs() << "Atomic load/store not supported yet\n"); 402 return false; 403 } 404 405 unsigned NewOpc = getLoadStoreOp(Ty, RB, Opc, MemOp.getAlignment()); 406 if (NewOpc == Opc) 407 return false; 408 409 X86AddressMode AM; 410 X86SelectAddress(*MRI.getVRegDef(I.getOperand(1).getReg()), MRI, AM); 411 412 I.setDesc(TII.get(NewOpc)); 413 MachineInstrBuilder MIB(MF, I); 414 if (Opc == TargetOpcode::G_LOAD) { 415 I.RemoveOperand(1); 416 addFullAddress(MIB, AM); 417 } else { 418 // G_STORE (VAL, Addr), X86Store instruction (Addr, VAL) 419 I.RemoveOperand(1); 420 I.RemoveOperand(0); 421 addFullAddress(MIB, AM).addUse(DefReg); 422 } 423 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 424 } 425 426 bool X86InstructionSelector::selectFrameIndexOrGep(MachineInstr &I, 427 MachineRegisterInfo &MRI, 428 MachineFunction &MF) const { 429 unsigned Opc = I.getOpcode(); 430 431 if (Opc != TargetOpcode::G_FRAME_INDEX && Opc != TargetOpcode::G_GEP) 432 return false; 433 434 const unsigned DefReg = I.getOperand(0).getReg(); 435 LLT Ty = MRI.getType(DefReg); 436 437 // Use LEA to calculate frame index and GEP 438 unsigned NewOpc; 439 if (Ty == LLT::pointer(0, 64)) 440 NewOpc = X86::LEA64r; 441 else if (Ty == LLT::pointer(0, 32)) 442 NewOpc = STI.isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r; 443 else 444 llvm_unreachable("Can't select G_FRAME_INDEX/G_GEP, unsupported type."); 445 446 I.setDesc(TII.get(NewOpc)); 447 MachineInstrBuilder MIB(MF, I); 448 449 if (Opc == TargetOpcode::G_FRAME_INDEX) { 450 addOffset(MIB, 0); 451 } else { 452 MachineOperand &InxOp = I.getOperand(2); 453 I.addOperand(InxOp); // set IndexReg 454 InxOp.ChangeToImmediate(1); // set Scale 455 MIB.addImm(0).addReg(0); 456 } 457 458 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 459 } 460 461 bool X86InstructionSelector::selectConstant(MachineInstr &I, 462 MachineRegisterInfo &MRI, 463 MachineFunction &MF) const { 464 if (I.getOpcode() != TargetOpcode::G_CONSTANT) 465 return false; 466 467 const unsigned DefReg = I.getOperand(0).getReg(); 468 LLT Ty = MRI.getType(DefReg); 469 470 assert(Ty.isScalar() && "invalid element type."); 471 472 uint64_t Val = 0; 473 if (I.getOperand(1).isCImm()) { 474 Val = I.getOperand(1).getCImm()->getZExtValue(); 475 I.getOperand(1).ChangeToImmediate(Val); 476 } else if (I.getOperand(1).isImm()) { 477 Val = I.getOperand(1).getImm(); 478 } else 479 llvm_unreachable("Unsupported operand type."); 480 481 unsigned NewOpc; 482 switch (Ty.getSizeInBits()) { 483 case 8: 484 NewOpc = X86::MOV8ri; 485 break; 486 case 16: 487 NewOpc = X86::MOV16ri; 488 break; 489 case 32: 490 NewOpc = X86::MOV32ri; 491 break; 492 case 64: { 493 // TODO: in case isUInt<32>(Val), X86::MOV32ri can be used 494 if (isInt<32>(Val)) 495 NewOpc = X86::MOV64ri32; 496 else 497 NewOpc = X86::MOV64ri; 498 break; 499 } 500 default: 501 llvm_unreachable("Can't select G_CONSTANT, unsupported type."); 502 } 503 504 I.setDesc(TII.get(NewOpc)); 505 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 506 } 507 508 bool X86InstructionSelector::selectTrunc(MachineInstr &I, 509 MachineRegisterInfo &MRI, 510 MachineFunction &MF) const { 511 if (I.getOpcode() != TargetOpcode::G_TRUNC) 512 return false; 513 514 const unsigned DstReg = I.getOperand(0).getReg(); 515 const unsigned SrcReg = I.getOperand(1).getReg(); 516 517 const LLT DstTy = MRI.getType(DstReg); 518 const LLT SrcTy = MRI.getType(SrcReg); 519 520 const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI); 521 const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI); 522 523 if (DstRB.getID() != SrcRB.getID()) { 524 DEBUG(dbgs() << "G_TRUNC input/output on different banks\n"); 525 return false; 526 } 527 528 if (DstRB.getID() != X86::GPRRegBankID) 529 return false; 530 531 const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB); 532 if (!DstRC) 533 return false; 534 535 const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB); 536 if (!SrcRC) 537 return false; 538 539 unsigned SubIdx; 540 if (DstRC == SrcRC) { 541 // Nothing to be done 542 SubIdx = X86::NoSubRegister; 543 } else if (DstRC == &X86::GR32RegClass) { 544 SubIdx = X86::sub_32bit; 545 } else if (DstRC == &X86::GR16RegClass) { 546 SubIdx = X86::sub_16bit; 547 } else if (DstRC == &X86::GR8RegClass) { 548 SubIdx = X86::sub_8bit; 549 } else { 550 return false; 551 } 552 553 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx); 554 555 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) || 556 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { 557 DEBUG(dbgs() << "Failed to constrain G_TRUNC\n"); 558 return false; 559 } 560 561 I.getOperand(1).setSubReg(SubIdx); 562 563 I.setDesc(TII.get(X86::COPY)); 564 return true; 565 } 566 567 bool X86InstructionSelector::selectZext(MachineInstr &I, 568 MachineRegisterInfo &MRI, 569 MachineFunction &MF) const { 570 if (I.getOpcode() != TargetOpcode::G_ZEXT) 571 return false; 572 573 const unsigned DstReg = I.getOperand(0).getReg(); 574 const unsigned SrcReg = I.getOperand(1).getReg(); 575 576 const LLT DstTy = MRI.getType(DstReg); 577 const LLT SrcTy = MRI.getType(SrcReg); 578 579 if (SrcTy == LLT::scalar(1)) { 580 581 unsigned AndOpc; 582 if (DstTy == LLT::scalar(32)) 583 AndOpc = X86::AND32ri8; 584 else if (DstTy == LLT::scalar(64)) 585 AndOpc = X86::AND64ri8; 586 else 587 return false; 588 589 unsigned DefReg = 590 MRI.createVirtualRegister(getRegClass(DstTy, DstReg, MRI)); 591 592 BuildMI(*I.getParent(), I, I.getDebugLoc(), 593 TII.get(TargetOpcode::SUBREG_TO_REG), DefReg) 594 .addImm(0) 595 .addReg(SrcReg) 596 .addImm(X86::sub_8bit); 597 598 MachineInstr &AndInst = 599 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AndOpc), DstReg) 600 .addReg(DefReg) 601 .addImm(1); 602 603 constrainSelectedInstRegOperands(AndInst, TII, TRI, RBI); 604 605 I.eraseFromParent(); 606 return true; 607 } 608 609 return false; 610 } 611 612 bool X86InstructionSelector::selectCmp(MachineInstr &I, 613 MachineRegisterInfo &MRI, 614 MachineFunction &MF) const { 615 if (I.getOpcode() != TargetOpcode::G_ICMP) 616 return false; 617 618 X86::CondCode CC; 619 bool SwapArgs; 620 std::tie(CC, SwapArgs) = X86::getX86ConditionCode( 621 (CmpInst::Predicate)I.getOperand(1).getPredicate()); 622 unsigned OpSet = X86::getSETFromCond(CC); 623 624 unsigned LHS = I.getOperand(2).getReg(); 625 unsigned RHS = I.getOperand(3).getReg(); 626 627 if (SwapArgs) 628 std::swap(LHS, RHS); 629 630 unsigned OpCmp; 631 LLT Ty = MRI.getType(LHS); 632 633 switch (Ty.getSizeInBits()) { 634 default: 635 return false; 636 case 8: 637 OpCmp = X86::CMP8rr; 638 break; 639 case 16: 640 OpCmp = X86::CMP16rr; 641 break; 642 case 32: 643 OpCmp = X86::CMP32rr; 644 break; 645 case 64: 646 OpCmp = X86::CMP64rr; 647 break; 648 } 649 650 MachineInstr &CmpInst = 651 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp)) 652 .addReg(LHS) 653 .addReg(RHS); 654 655 MachineInstr &SetInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(), 656 TII.get(OpSet), I.getOperand(0).getReg()); 657 658 constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI); 659 constrainSelectedInstRegOperands(SetInst, TII, TRI, RBI); 660 661 I.eraseFromParent(); 662 return true; 663 } 664 665 bool X86InstructionSelector::selectUadde(MachineInstr &I, 666 MachineRegisterInfo &MRI, 667 MachineFunction &MF) const { 668 if (I.getOpcode() != TargetOpcode::G_UADDE) 669 return false; 670 671 const unsigned DstReg = I.getOperand(0).getReg(); 672 const unsigned CarryOutReg = I.getOperand(1).getReg(); 673 const unsigned Op0Reg = I.getOperand(2).getReg(); 674 const unsigned Op1Reg = I.getOperand(3).getReg(); 675 unsigned CarryInReg = I.getOperand(4).getReg(); 676 677 const LLT DstTy = MRI.getType(DstReg); 678 679 if (DstTy != LLT::scalar(32)) 680 return false; 681 682 // find CarryIn def instruction. 683 MachineInstr *Def = MRI.getVRegDef(CarryInReg); 684 while (Def->getOpcode() == TargetOpcode::G_TRUNC) { 685 CarryInReg = Def->getOperand(1).getReg(); 686 Def = MRI.getVRegDef(CarryInReg); 687 } 688 689 unsigned Opcode; 690 if (Def->getOpcode() == TargetOpcode::G_UADDE) { 691 // carry set by prev ADD. 692 693 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), X86::EFLAGS) 694 .addReg(CarryInReg); 695 696 if (!RBI.constrainGenericRegister(CarryInReg, X86::GR32RegClass, MRI)) 697 return false; 698 699 Opcode = X86::ADC32rr; 700 } else if (auto val = getConstantVRegVal(CarryInReg, MRI)) { 701 // carry is constant, support only 0. 702 if (*val != 0) 703 return false; 704 705 Opcode = X86::ADD32rr; 706 } else 707 return false; 708 709 MachineInstr &AddInst = 710 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode), DstReg) 711 .addReg(Op0Reg) 712 .addReg(Op1Reg); 713 714 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), CarryOutReg) 715 .addReg(X86::EFLAGS); 716 717 if (!constrainSelectedInstRegOperands(AddInst, TII, TRI, RBI) || 718 !RBI.constrainGenericRegister(CarryOutReg, X86::GR32RegClass, MRI)) 719 return false; 720 721 I.eraseFromParent(); 722 return true; 723 } 724 725 bool X86InstructionSelector::selectExtract(MachineInstr &I, 726 MachineRegisterInfo &MRI, 727 MachineFunction &MF) const { 728 729 if (I.getOpcode() != TargetOpcode::G_EXTRACT) 730 return false; 731 732 const unsigned DstReg = I.getOperand(0).getReg(); 733 const unsigned SrcReg = I.getOperand(1).getReg(); 734 int64_t Index = I.getOperand(2).getImm(); 735 736 const LLT DstTy = MRI.getType(DstReg); 737 const LLT SrcTy = MRI.getType(SrcReg); 738 739 // Meanwile handle vector type only. 740 if (!DstTy.isVector()) 741 return false; 742 743 if (Index % DstTy.getSizeInBits() != 0) 744 return false; // Not extract subvector. 745 746 if (Index == 0) { 747 // Replace by extract subreg copy. 748 if (!emitExtractSubreg(DstReg, SrcReg, I, MRI, MF)) 749 return false; 750 751 I.eraseFromParent(); 752 return true; 753 } 754 755 bool HasAVX = STI.hasAVX(); 756 bool HasAVX512 = STI.hasAVX512(); 757 bool HasVLX = STI.hasVLX(); 758 759 if (SrcTy.getSizeInBits() == 256 && DstTy.getSizeInBits() == 128) { 760 if (HasVLX) 761 I.setDesc(TII.get(X86::VEXTRACTF32x4Z256rr)); 762 else if (HasAVX) 763 I.setDesc(TII.get(X86::VEXTRACTF128rr)); 764 else 765 return false; 766 } else if (SrcTy.getSizeInBits() == 512 && HasAVX512) { 767 if (DstTy.getSizeInBits() == 128) 768 I.setDesc(TII.get(X86::VEXTRACTF32x4Zrr)); 769 else if (DstTy.getSizeInBits() == 256) 770 I.setDesc(TII.get(X86::VEXTRACTF64x4Zrr)); 771 else 772 return false; 773 } else 774 return false; 775 776 // Convert to X86 VEXTRACT immediate. 777 Index = Index / DstTy.getSizeInBits(); 778 I.getOperand(2).setImm(Index); 779 780 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 781 } 782 783 bool X86InstructionSelector::emitExtractSubreg(unsigned DstReg, unsigned SrcReg, 784 MachineInstr &I, 785 MachineRegisterInfo &MRI, 786 MachineFunction &MF) const { 787 788 const LLT DstTy = MRI.getType(DstReg); 789 const LLT SrcTy = MRI.getType(SrcReg); 790 unsigned SubIdx = X86::NoSubRegister; 791 792 if (!DstTy.isVector() || !SrcTy.isVector()) 793 return false; 794 795 assert(SrcTy.getSizeInBits() > DstTy.getSizeInBits() && 796 "Incorrect Src/Dst register size"); 797 798 if (DstTy.getSizeInBits() == 128) 799 SubIdx = X86::sub_xmm; 800 else if (DstTy.getSizeInBits() == 256) 801 SubIdx = X86::sub_ymm; 802 else 803 return false; 804 805 const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI); 806 const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI); 807 808 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx); 809 810 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) || 811 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { 812 DEBUG(dbgs() << "Failed to constrain G_TRUNC\n"); 813 return false; 814 } 815 816 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), DstReg) 817 .addReg(SrcReg, 0, SubIdx); 818 819 return true; 820 } 821 822 bool X86InstructionSelector::emitInsertSubreg(unsigned DstReg, unsigned SrcReg, 823 MachineInstr &I, 824 MachineRegisterInfo &MRI, 825 MachineFunction &MF) const { 826 827 const LLT DstTy = MRI.getType(DstReg); 828 const LLT SrcTy = MRI.getType(SrcReg); 829 unsigned SubIdx = X86::NoSubRegister; 830 831 // TODO: support scalar types 832 if (!DstTy.isVector() || !SrcTy.isVector()) 833 return false; 834 835 assert(SrcTy.getSizeInBits() < DstTy.getSizeInBits() && 836 "Incorrect Src/Dst register size"); 837 838 if (SrcTy.getSizeInBits() == 128) 839 SubIdx = X86::sub_xmm; 840 else if (SrcTy.getSizeInBits() == 256) 841 SubIdx = X86::sub_ymm; 842 else 843 return false; 844 845 const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI); 846 const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI); 847 848 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) || 849 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { 850 DEBUG(dbgs() << "Failed to constrain INSERT_SUBREG\n"); 851 return false; 852 } 853 854 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY)) 855 .addReg(DstReg, RegState::DefineNoRead, SubIdx) 856 .addReg(SrcReg); 857 858 return true; 859 } 860 861 bool X86InstructionSelector::selectInsert(MachineInstr &I, 862 MachineRegisterInfo &MRI, 863 MachineFunction &MF) const { 864 865 if (I.getOpcode() != TargetOpcode::G_INSERT) 866 return false; 867 868 const unsigned DstReg = I.getOperand(0).getReg(); 869 const unsigned SrcReg = I.getOperand(1).getReg(); 870 const unsigned InsertReg = I.getOperand(2).getReg(); 871 int64_t Index = I.getOperand(3).getImm(); 872 873 const LLT DstTy = MRI.getType(DstReg); 874 const LLT InsertRegTy = MRI.getType(InsertReg); 875 876 // Meanwile handle vector type only. 877 if (!DstTy.isVector()) 878 return false; 879 880 if (Index % InsertRegTy.getSizeInBits() != 0) 881 return false; // Not insert subvector. 882 883 if (Index == 0 && MRI.getVRegDef(SrcReg)->isImplicitDef()) { 884 // Replace by subreg copy. 885 if (!emitInsertSubreg(DstReg, InsertReg, I, MRI, MF)) 886 return false; 887 888 I.eraseFromParent(); 889 return true; 890 } 891 892 bool HasAVX = STI.hasAVX(); 893 bool HasAVX512 = STI.hasAVX512(); 894 bool HasVLX = STI.hasVLX(); 895 896 if (DstTy.getSizeInBits() == 256 && InsertRegTy.getSizeInBits() == 128) { 897 if (HasVLX) 898 I.setDesc(TII.get(X86::VINSERTF32x4Z256rr)); 899 else if (HasAVX) 900 I.setDesc(TII.get(X86::VINSERTF128rr)); 901 else 902 return false; 903 } else if (DstTy.getSizeInBits() == 512 && HasAVX512) { 904 if (InsertRegTy.getSizeInBits() == 128) 905 I.setDesc(TII.get(X86::VINSERTF32x4Zrr)); 906 else if (InsertRegTy.getSizeInBits() == 256) 907 I.setDesc(TII.get(X86::VINSERTF64x4Zrr)); 908 else 909 return false; 910 } else 911 return false; 912 913 // Convert to X86 VINSERT immediate. 914 Index = Index / InsertRegTy.getSizeInBits(); 915 916 I.getOperand(3).setImm(Index); 917 918 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 919 } 920 921 bool X86InstructionSelector::selectMergeValues(MachineInstr &I, 922 MachineRegisterInfo &MRI, 923 MachineFunction &MF) const { 924 if (I.getOpcode() != TargetOpcode::G_MERGE_VALUES) 925 return false; 926 927 // Split to inserts. 928 unsigned DstReg = I.getOperand(0).getReg(); 929 unsigned SrcReg0 = I.getOperand(1).getReg(); 930 931 const LLT DstTy = MRI.getType(DstReg); 932 const LLT SrcTy = MRI.getType(SrcReg0); 933 unsigned SrcSize = SrcTy.getSizeInBits(); 934 935 const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI); 936 937 // For the first src use insertSubReg. 938 unsigned DefReg = MRI.createGenericVirtualRegister(DstTy); 939 MRI.setRegBank(DefReg, RegBank); 940 if (!emitInsertSubreg(DefReg, I.getOperand(1).getReg(), I, MRI, MF)) 941 return false; 942 943 for (unsigned Idx = 2; Idx < I.getNumOperands(); ++Idx) { 944 945 unsigned Tmp = MRI.createGenericVirtualRegister(DstTy); 946 MRI.setRegBank(Tmp, RegBank); 947 948 MachineInstr &InsertInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(), 949 TII.get(TargetOpcode::G_INSERT), Tmp) 950 .addReg(DefReg) 951 .addReg(I.getOperand(Idx).getReg()) 952 .addImm((Idx - 1) * SrcSize); 953 954 DefReg = Tmp; 955 956 if (!select(InsertInst)) 957 return false; 958 } 959 960 MachineInstr &CopyInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(), 961 TII.get(TargetOpcode::COPY), DstReg) 962 .addReg(DefReg); 963 964 if (!select(CopyInst)) 965 return false; 966 967 I.eraseFromParent(); 968 return true; 969 } 970 InstructionSelector * 971 llvm::createX86InstructionSelector(const X86TargetMachine &TM, 972 X86Subtarget &Subtarget, 973 X86RegisterBankInfo &RBI) { 974 return new X86InstructionSelector(TM, Subtarget, RBI); 975 } 976