1 //===- X86InstructionSelector.cpp ----------------------------*- C++ -*-==// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 /// \file 10 /// This file implements the targeting of the InstructionSelector class for 11 /// X86. 12 /// \todo This should be generated by TableGen. 13 //===----------------------------------------------------------------------===// 14 15 #include "X86InstrBuilder.h" 16 #include "X86InstrInfo.h" 17 #include "X86RegisterBankInfo.h" 18 #include "X86RegisterInfo.h" 19 #include "X86Subtarget.h" 20 #include "X86TargetMachine.h" 21 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 22 #include "llvm/CodeGen/GlobalISel/Utils.h" 23 #include "llvm/CodeGen/MachineBasicBlock.h" 24 #include "llvm/CodeGen/MachineFunction.h" 25 #include "llvm/CodeGen/MachineInstr.h" 26 #include "llvm/CodeGen/MachineInstrBuilder.h" 27 #include "llvm/CodeGen/MachineOperand.h" 28 #include "llvm/CodeGen/MachineRegisterInfo.h" 29 #include "llvm/IR/Type.h" 30 #include "llvm/Support/Debug.h" 31 #include "llvm/Support/raw_ostream.h" 32 33 #define DEBUG_TYPE "X86-isel" 34 35 #include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h" 36 37 using namespace llvm; 38 39 #ifndef LLVM_BUILD_GLOBAL_ISEL 40 #error "You shouldn't build this" 41 #endif 42 43 namespace { 44 45 #define GET_GLOBALISEL_PREDICATE_BITSET 46 #include "X86GenGlobalISel.inc" 47 #undef GET_GLOBALISEL_PREDICATE_BITSET 48 49 class X86InstructionSelector : public InstructionSelector { 50 public: 51 X86InstructionSelector(const X86TargetMachine &TM, const X86Subtarget &STI, 52 const X86RegisterBankInfo &RBI); 53 54 bool select(MachineInstr &I) const override; 55 56 private: 57 /// tblgen-erated 'select' implementation, used as the initial selector for 58 /// the patterns that don't require complex C++. 59 bool selectImpl(MachineInstr &I) const; 60 61 // TODO: remove after supported by Tablegen-erated instruction selection. 62 unsigned getLoadStoreOp(LLT &Ty, const RegisterBank &RB, unsigned Opc, 63 uint64_t Alignment) const; 64 65 bool selectLoadStoreOp(MachineInstr &I, MachineRegisterInfo &MRI, 66 MachineFunction &MF) const; 67 bool selectFrameIndexOrGep(MachineInstr &I, MachineRegisterInfo &MRI, 68 MachineFunction &MF) const; 69 bool selectGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI, 70 MachineFunction &MF) const; 71 bool selectConstant(MachineInstr &I, MachineRegisterInfo &MRI, 72 MachineFunction &MF) const; 73 bool selectTrunc(MachineInstr &I, MachineRegisterInfo &MRI, 74 MachineFunction &MF) const; 75 bool selectZext(MachineInstr &I, MachineRegisterInfo &MRI, 76 MachineFunction &MF) const; 77 bool selectCmp(MachineInstr &I, MachineRegisterInfo &MRI, 78 MachineFunction &MF) const; 79 bool selectUadde(MachineInstr &I, MachineRegisterInfo &MRI, 80 MachineFunction &MF) const; 81 bool selectCopy(MachineInstr &I, MachineRegisterInfo &MRI) const; 82 bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI, 83 MachineFunction &MF) const; 84 bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI, 85 MachineFunction &MF) const; 86 bool selectInsert(MachineInstr &I, MachineRegisterInfo &MRI, 87 MachineFunction &MF) const; 88 bool selectExtract(MachineInstr &I, MachineRegisterInfo &MRI, 89 MachineFunction &MF) const; 90 91 // emit insert subreg instruction and insert it before MachineInstr &I 92 bool emitInsertSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I, 93 MachineRegisterInfo &MRI, MachineFunction &MF) const; 94 // emit extract subreg instruction and insert it before MachineInstr &I 95 bool emitExtractSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I, 96 MachineRegisterInfo &MRI, MachineFunction &MF) const; 97 98 const TargetRegisterClass *getRegClass(LLT Ty, const RegisterBank &RB) const; 99 const TargetRegisterClass *getRegClass(LLT Ty, unsigned Reg, 100 MachineRegisterInfo &MRI) const; 101 102 const X86TargetMachine &TM; 103 const X86Subtarget &STI; 104 const X86InstrInfo &TII; 105 const X86RegisterInfo &TRI; 106 const X86RegisterBankInfo &RBI; 107 108 #define GET_GLOBALISEL_PREDICATES_DECL 109 #include "X86GenGlobalISel.inc" 110 #undef GET_GLOBALISEL_PREDICATES_DECL 111 112 #define GET_GLOBALISEL_TEMPORARIES_DECL 113 #include "X86GenGlobalISel.inc" 114 #undef GET_GLOBALISEL_TEMPORARIES_DECL 115 }; 116 117 } // end anonymous namespace 118 119 #define GET_GLOBALISEL_IMPL 120 #include "X86GenGlobalISel.inc" 121 #undef GET_GLOBALISEL_IMPL 122 123 X86InstructionSelector::X86InstructionSelector(const X86TargetMachine &TM, 124 const X86Subtarget &STI, 125 const X86RegisterBankInfo &RBI) 126 : InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()), 127 TRI(*STI.getRegisterInfo()), RBI(RBI), 128 #define GET_GLOBALISEL_PREDICATES_INIT 129 #include "X86GenGlobalISel.inc" 130 #undef GET_GLOBALISEL_PREDICATES_INIT 131 #define GET_GLOBALISEL_TEMPORARIES_INIT 132 #include "X86GenGlobalISel.inc" 133 #undef GET_GLOBALISEL_TEMPORARIES_INIT 134 { 135 } 136 137 // FIXME: This should be target-independent, inferred from the types declared 138 // for each class in the bank. 139 const TargetRegisterClass * 140 X86InstructionSelector::getRegClass(LLT Ty, const RegisterBank &RB) const { 141 if (RB.getID() == X86::GPRRegBankID) { 142 if (Ty.getSizeInBits() <= 8) 143 return &X86::GR8RegClass; 144 if (Ty.getSizeInBits() == 16) 145 return &X86::GR16RegClass; 146 if (Ty.getSizeInBits() == 32) 147 return &X86::GR32RegClass; 148 if (Ty.getSizeInBits() == 64) 149 return &X86::GR64RegClass; 150 } 151 if (RB.getID() == X86::VECRRegBankID) { 152 if (Ty.getSizeInBits() == 32) 153 return STI.hasAVX512() ? &X86::FR32XRegClass : &X86::FR32RegClass; 154 if (Ty.getSizeInBits() == 64) 155 return STI.hasAVX512() ? &X86::FR64XRegClass : &X86::FR64RegClass; 156 if (Ty.getSizeInBits() == 128) 157 return STI.hasAVX512() ? &X86::VR128XRegClass : &X86::VR128RegClass; 158 if (Ty.getSizeInBits() == 256) 159 return STI.hasAVX512() ? &X86::VR256XRegClass : &X86::VR256RegClass; 160 if (Ty.getSizeInBits() == 512) 161 return &X86::VR512RegClass; 162 } 163 164 llvm_unreachable("Unknown RegBank!"); 165 } 166 167 const TargetRegisterClass * 168 X86InstructionSelector::getRegClass(LLT Ty, unsigned Reg, 169 MachineRegisterInfo &MRI) const { 170 const RegisterBank &RegBank = *RBI.getRegBank(Reg, MRI, TRI); 171 return getRegClass(Ty, RegBank); 172 } 173 174 // Set X86 Opcode and constrain DestReg. 175 bool X86InstructionSelector::selectCopy(MachineInstr &I, 176 MachineRegisterInfo &MRI) const { 177 178 unsigned DstReg = I.getOperand(0).getReg(); 179 if (TargetRegisterInfo::isPhysicalRegister(DstReg)) { 180 assert(I.isCopy() && "Generic operators do not allow physical registers"); 181 return true; 182 } 183 184 const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI); 185 const unsigned DstSize = MRI.getType(DstReg).getSizeInBits(); 186 unsigned SrcReg = I.getOperand(1).getReg(); 187 const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI); 188 189 assert((!TargetRegisterInfo::isPhysicalRegister(SrcReg) || I.isCopy()) && 190 "No phys reg on generic operators"); 191 assert((DstSize == SrcSize || 192 // Copies are a mean to setup initial types, the number of 193 // bits may not exactly match. 194 (TargetRegisterInfo::isPhysicalRegister(SrcReg) && 195 DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI))) && 196 "Copy with different width?!"); 197 198 const TargetRegisterClass *RC = nullptr; 199 200 switch (RegBank.getID()) { 201 case X86::GPRRegBankID: 202 assert((DstSize <= 64) && "GPRs cannot get more than 64-bit width values."); 203 RC = getRegClass(MRI.getType(DstReg), RegBank); 204 205 // Change the physical register 206 if (SrcSize > DstSize && TargetRegisterInfo::isPhysicalRegister(SrcReg)) { 207 if (RC == &X86::GR32RegClass) 208 I.getOperand(1).setSubReg(X86::sub_32bit); 209 else if (RC == &X86::GR16RegClass) 210 I.getOperand(1).setSubReg(X86::sub_16bit); 211 else if (RC == &X86::GR8RegClass) 212 I.getOperand(1).setSubReg(X86::sub_8bit); 213 214 I.getOperand(1).substPhysReg(SrcReg, TRI); 215 } 216 break; 217 case X86::VECRRegBankID: 218 RC = getRegClass(MRI.getType(DstReg), RegBank); 219 break; 220 default: 221 llvm_unreachable("Unknown RegBank!"); 222 } 223 224 // No need to constrain SrcReg. It will get constrained when 225 // we hit another of its use or its defs. 226 // Copies do not have constraints. 227 const TargetRegisterClass *OldRC = MRI.getRegClassOrNull(DstReg); 228 if (!OldRC || !RC->hasSubClassEq(OldRC)) { 229 if (!RBI.constrainGenericRegister(DstReg, *RC, MRI)) { 230 DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode()) 231 << " operand\n"); 232 return false; 233 } 234 } 235 I.setDesc(TII.get(X86::COPY)); 236 return true; 237 } 238 239 bool X86InstructionSelector::select(MachineInstr &I) const { 240 assert(I.getParent() && "Instruction should be in a basic block!"); 241 assert(I.getParent()->getParent() && "Instruction should be in a function!"); 242 243 MachineBasicBlock &MBB = *I.getParent(); 244 MachineFunction &MF = *MBB.getParent(); 245 MachineRegisterInfo &MRI = MF.getRegInfo(); 246 247 unsigned Opcode = I.getOpcode(); 248 if (!isPreISelGenericOpcode(Opcode)) { 249 // Certain non-generic instructions also need some special handling. 250 251 if (I.isCopy()) 252 return selectCopy(I, MRI); 253 254 // TODO: handle more cases - LOAD_STACK_GUARD, PHI 255 return true; 256 } 257 258 assert(I.getNumOperands() == I.getNumExplicitOperands() && 259 "Generic instruction has unexpected implicit operands\n"); 260 261 if (selectImpl(I)) 262 return true; 263 264 DEBUG(dbgs() << " C++ instruction selection: "; I.print(dbgs())); 265 266 // TODO: This should be implemented by tblgen. 267 if (selectLoadStoreOp(I, MRI, MF)) 268 return true; 269 if (selectFrameIndexOrGep(I, MRI, MF)) 270 return true; 271 if (selectGlobalValue(I, MRI, MF)) 272 return true; 273 if (selectConstant(I, MRI, MF)) 274 return true; 275 if (selectTrunc(I, MRI, MF)) 276 return true; 277 if (selectZext(I, MRI, MF)) 278 return true; 279 if (selectCmp(I, MRI, MF)) 280 return true; 281 if (selectUadde(I, MRI, MF)) 282 return true; 283 if (selectUnmergeValues(I, MRI, MF)) 284 return true; 285 if (selectMergeValues(I, MRI, MF)) 286 return true; 287 if (selectExtract(I, MRI, MF)) 288 return true; 289 if (selectInsert(I, MRI, MF)) 290 return true; 291 292 return false; 293 } 294 295 unsigned X86InstructionSelector::getLoadStoreOp(LLT &Ty, const RegisterBank &RB, 296 unsigned Opc, 297 uint64_t Alignment) const { 298 bool Isload = (Opc == TargetOpcode::G_LOAD); 299 bool HasAVX = STI.hasAVX(); 300 bool HasAVX512 = STI.hasAVX512(); 301 bool HasVLX = STI.hasVLX(); 302 303 if (Ty == LLT::scalar(8)) { 304 if (X86::GPRRegBankID == RB.getID()) 305 return Isload ? X86::MOV8rm : X86::MOV8mr; 306 } else if (Ty == LLT::scalar(16)) { 307 if (X86::GPRRegBankID == RB.getID()) 308 return Isload ? X86::MOV16rm : X86::MOV16mr; 309 } else if (Ty == LLT::scalar(32) || Ty == LLT::pointer(0, 32)) { 310 if (X86::GPRRegBankID == RB.getID()) 311 return Isload ? X86::MOV32rm : X86::MOV32mr; 312 if (X86::VECRRegBankID == RB.getID()) 313 return Isload ? (HasAVX512 ? X86::VMOVSSZrm 314 : HasAVX ? X86::VMOVSSrm : X86::MOVSSrm) 315 : (HasAVX512 ? X86::VMOVSSZmr 316 : HasAVX ? X86::VMOVSSmr : X86::MOVSSmr); 317 } else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64)) { 318 if (X86::GPRRegBankID == RB.getID()) 319 return Isload ? X86::MOV64rm : X86::MOV64mr; 320 if (X86::VECRRegBankID == RB.getID()) 321 return Isload ? (HasAVX512 ? X86::VMOVSDZrm 322 : HasAVX ? X86::VMOVSDrm : X86::MOVSDrm) 323 : (HasAVX512 ? X86::VMOVSDZmr 324 : HasAVX ? X86::VMOVSDmr : X86::MOVSDmr); 325 } else if (Ty.isVector() && Ty.getSizeInBits() == 128) { 326 if (Alignment >= 16) 327 return Isload ? (HasVLX ? X86::VMOVAPSZ128rm 328 : HasAVX512 329 ? X86::VMOVAPSZ128rm_NOVLX 330 : HasAVX ? X86::VMOVAPSrm : X86::MOVAPSrm) 331 : (HasVLX ? X86::VMOVAPSZ128mr 332 : HasAVX512 333 ? X86::VMOVAPSZ128mr_NOVLX 334 : HasAVX ? X86::VMOVAPSmr : X86::MOVAPSmr); 335 else 336 return Isload ? (HasVLX ? X86::VMOVUPSZ128rm 337 : HasAVX512 338 ? X86::VMOVUPSZ128rm_NOVLX 339 : HasAVX ? X86::VMOVUPSrm : X86::MOVUPSrm) 340 : (HasVLX ? X86::VMOVUPSZ128mr 341 : HasAVX512 342 ? X86::VMOVUPSZ128mr_NOVLX 343 : HasAVX ? X86::VMOVUPSmr : X86::MOVUPSmr); 344 } else if (Ty.isVector() && Ty.getSizeInBits() == 256) { 345 if (Alignment >= 32) 346 return Isload ? (HasVLX ? X86::VMOVAPSZ256rm 347 : HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX 348 : X86::VMOVAPSYrm) 349 : (HasVLX ? X86::VMOVAPSZ256mr 350 : HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX 351 : X86::VMOVAPSYmr); 352 else 353 return Isload ? (HasVLX ? X86::VMOVUPSZ256rm 354 : HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX 355 : X86::VMOVUPSYrm) 356 : (HasVLX ? X86::VMOVUPSZ256mr 357 : HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX 358 : X86::VMOVUPSYmr); 359 } else if (Ty.isVector() && Ty.getSizeInBits() == 512) { 360 if (Alignment >= 64) 361 return Isload ? X86::VMOVAPSZrm : X86::VMOVAPSZmr; 362 else 363 return Isload ? X86::VMOVUPSZrm : X86::VMOVUPSZmr; 364 } 365 return Opc; 366 } 367 368 // Fill in an address from the given instruction. 369 void X86SelectAddress(const MachineInstr &I, const MachineRegisterInfo &MRI, 370 X86AddressMode &AM) { 371 372 assert(I.getOperand(0).isReg() && "unsupported opperand."); 373 assert(MRI.getType(I.getOperand(0).getReg()).isPointer() && 374 "unsupported type."); 375 376 if (I.getOpcode() == TargetOpcode::G_GEP) { 377 if (auto COff = getConstantVRegVal(I.getOperand(2).getReg(), MRI)) { 378 int64_t Imm = *COff; 379 if (isInt<32>(Imm)) { // Check for displacement overflow. 380 AM.Disp = static_cast<int32_t>(Imm); 381 AM.Base.Reg = I.getOperand(1).getReg(); 382 return; 383 } 384 } 385 } else if (I.getOpcode() == TargetOpcode::G_FRAME_INDEX) { 386 AM.Base.FrameIndex = I.getOperand(1).getIndex(); 387 AM.BaseType = X86AddressMode::FrameIndexBase; 388 return; 389 } 390 391 // Default behavior. 392 AM.Base.Reg = I.getOperand(0).getReg(); 393 return; 394 } 395 396 bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I, 397 MachineRegisterInfo &MRI, 398 MachineFunction &MF) const { 399 400 unsigned Opc = I.getOpcode(); 401 402 if (Opc != TargetOpcode::G_STORE && Opc != TargetOpcode::G_LOAD) 403 return false; 404 405 const unsigned DefReg = I.getOperand(0).getReg(); 406 LLT Ty = MRI.getType(DefReg); 407 const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI); 408 409 auto &MemOp = **I.memoperands_begin(); 410 if (MemOp.getOrdering() != AtomicOrdering::NotAtomic) { 411 DEBUG(dbgs() << "Atomic load/store not supported yet\n"); 412 return false; 413 } 414 415 unsigned NewOpc = getLoadStoreOp(Ty, RB, Opc, MemOp.getAlignment()); 416 if (NewOpc == Opc) 417 return false; 418 419 X86AddressMode AM; 420 X86SelectAddress(*MRI.getVRegDef(I.getOperand(1).getReg()), MRI, AM); 421 422 I.setDesc(TII.get(NewOpc)); 423 MachineInstrBuilder MIB(MF, I); 424 if (Opc == TargetOpcode::G_LOAD) { 425 I.RemoveOperand(1); 426 addFullAddress(MIB, AM); 427 } else { 428 // G_STORE (VAL, Addr), X86Store instruction (Addr, VAL) 429 I.RemoveOperand(1); 430 I.RemoveOperand(0); 431 addFullAddress(MIB, AM).addUse(DefReg); 432 } 433 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 434 } 435 436 static unsigned getLeaOP(LLT Ty, const X86Subtarget &STI) { 437 if (Ty == LLT::pointer(0, 64)) 438 return X86::LEA64r; 439 else if (Ty == LLT::pointer(0, 32)) 440 return STI.isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r; 441 else 442 llvm_unreachable("Can't get LEA opcode. Unsupported type."); 443 } 444 445 bool X86InstructionSelector::selectFrameIndexOrGep(MachineInstr &I, 446 MachineRegisterInfo &MRI, 447 MachineFunction &MF) const { 448 unsigned Opc = I.getOpcode(); 449 450 if (Opc != TargetOpcode::G_FRAME_INDEX && Opc != TargetOpcode::G_GEP) 451 return false; 452 453 const unsigned DefReg = I.getOperand(0).getReg(); 454 LLT Ty = MRI.getType(DefReg); 455 456 // Use LEA to calculate frame index and GEP 457 unsigned NewOpc = getLeaOP(Ty, STI); 458 I.setDesc(TII.get(NewOpc)); 459 MachineInstrBuilder MIB(MF, I); 460 461 if (Opc == TargetOpcode::G_FRAME_INDEX) { 462 addOffset(MIB, 0); 463 } else { 464 MachineOperand &InxOp = I.getOperand(2); 465 I.addOperand(InxOp); // set IndexReg 466 InxOp.ChangeToImmediate(1); // set Scale 467 MIB.addImm(0).addReg(0); 468 } 469 470 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 471 } 472 473 bool X86InstructionSelector::selectGlobalValue(MachineInstr &I, 474 MachineRegisterInfo &MRI, 475 MachineFunction &MF) const { 476 unsigned Opc = I.getOpcode(); 477 478 if (Opc != TargetOpcode::G_GLOBAL_VALUE) 479 return false; 480 481 auto GV = I.getOperand(1).getGlobal(); 482 if (GV->isThreadLocal()) { 483 return false; // TODO: we don't support TLS yet. 484 } 485 486 // Can't handle alternate code models yet. 487 if (TM.getCodeModel() != CodeModel::Small) 488 return 0; 489 490 X86AddressMode AM; 491 AM.GV = GV; 492 AM.GVOpFlags = STI.classifyGlobalReference(GV); 493 494 // TODO: The ABI requires an extra load. not supported yet. 495 if (isGlobalStubReference(AM.GVOpFlags)) 496 return false; 497 498 // TODO: This reference is relative to the pic base. not supported yet. 499 if (isGlobalRelativeToPICBase(AM.GVOpFlags)) 500 return false; 501 502 if (STI.isPICStyleRIPRel()) { 503 // Use rip-relative addressing. 504 assert(AM.Base.Reg == 0 && AM.IndexReg == 0); 505 AM.Base.Reg = X86::RIP; 506 } 507 508 const unsigned DefReg = I.getOperand(0).getReg(); 509 LLT Ty = MRI.getType(DefReg); 510 unsigned NewOpc = getLeaOP(Ty, STI); 511 512 I.setDesc(TII.get(NewOpc)); 513 MachineInstrBuilder MIB(MF, I); 514 515 I.RemoveOperand(1); 516 addFullAddress(MIB, AM); 517 518 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 519 } 520 521 bool X86InstructionSelector::selectConstant(MachineInstr &I, 522 MachineRegisterInfo &MRI, 523 MachineFunction &MF) const { 524 if (I.getOpcode() != TargetOpcode::G_CONSTANT) 525 return false; 526 527 const unsigned DefReg = I.getOperand(0).getReg(); 528 LLT Ty = MRI.getType(DefReg); 529 530 if (RBI.getRegBank(DefReg, MRI, TRI)->getID() != X86::GPRRegBankID) 531 return false; 532 533 uint64_t Val = 0; 534 if (I.getOperand(1).isCImm()) { 535 Val = I.getOperand(1).getCImm()->getZExtValue(); 536 I.getOperand(1).ChangeToImmediate(Val); 537 } else if (I.getOperand(1).isImm()) { 538 Val = I.getOperand(1).getImm(); 539 } else 540 llvm_unreachable("Unsupported operand type."); 541 542 unsigned NewOpc; 543 switch (Ty.getSizeInBits()) { 544 case 8: 545 NewOpc = X86::MOV8ri; 546 break; 547 case 16: 548 NewOpc = X86::MOV16ri; 549 break; 550 case 32: 551 NewOpc = X86::MOV32ri; 552 break; 553 case 64: { 554 // TODO: in case isUInt<32>(Val), X86::MOV32ri can be used 555 if (isInt<32>(Val)) 556 NewOpc = X86::MOV64ri32; 557 else 558 NewOpc = X86::MOV64ri; 559 break; 560 } 561 default: 562 llvm_unreachable("Can't select G_CONSTANT, unsupported type."); 563 } 564 565 I.setDesc(TII.get(NewOpc)); 566 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 567 } 568 569 bool X86InstructionSelector::selectTrunc(MachineInstr &I, 570 MachineRegisterInfo &MRI, 571 MachineFunction &MF) const { 572 if (I.getOpcode() != TargetOpcode::G_TRUNC) 573 return false; 574 575 const unsigned DstReg = I.getOperand(0).getReg(); 576 const unsigned SrcReg = I.getOperand(1).getReg(); 577 578 const LLT DstTy = MRI.getType(DstReg); 579 const LLT SrcTy = MRI.getType(SrcReg); 580 581 const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI); 582 const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI); 583 584 if (DstRB.getID() != SrcRB.getID()) { 585 DEBUG(dbgs() << "G_TRUNC input/output on different banks\n"); 586 return false; 587 } 588 589 if (DstRB.getID() != X86::GPRRegBankID) 590 return false; 591 592 const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB); 593 if (!DstRC) 594 return false; 595 596 const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB); 597 if (!SrcRC) 598 return false; 599 600 unsigned SubIdx; 601 if (DstRC == SrcRC) { 602 // Nothing to be done 603 SubIdx = X86::NoSubRegister; 604 } else if (DstRC == &X86::GR32RegClass) { 605 SubIdx = X86::sub_32bit; 606 } else if (DstRC == &X86::GR16RegClass) { 607 SubIdx = X86::sub_16bit; 608 } else if (DstRC == &X86::GR8RegClass) { 609 SubIdx = X86::sub_8bit; 610 } else { 611 return false; 612 } 613 614 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx); 615 616 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) || 617 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { 618 DEBUG(dbgs() << "Failed to constrain G_TRUNC\n"); 619 return false; 620 } 621 622 I.getOperand(1).setSubReg(SubIdx); 623 624 I.setDesc(TII.get(X86::COPY)); 625 return true; 626 } 627 628 bool X86InstructionSelector::selectZext(MachineInstr &I, 629 MachineRegisterInfo &MRI, 630 MachineFunction &MF) const { 631 if (I.getOpcode() != TargetOpcode::G_ZEXT) 632 return false; 633 634 const unsigned DstReg = I.getOperand(0).getReg(); 635 const unsigned SrcReg = I.getOperand(1).getReg(); 636 637 const LLT DstTy = MRI.getType(DstReg); 638 const LLT SrcTy = MRI.getType(SrcReg); 639 640 if (SrcTy != LLT::scalar(1)) 641 return false; 642 643 unsigned AndOpc; 644 if (DstTy == LLT::scalar(8)) 645 AndOpc = X86::AND8ri; 646 else if (DstTy == LLT::scalar(16)) 647 AndOpc = X86::AND16ri8; 648 else if (DstTy == LLT::scalar(32)) 649 AndOpc = X86::AND32ri8; 650 else if (DstTy == LLT::scalar(64)) 651 AndOpc = X86::AND64ri8; 652 else 653 return false; 654 655 unsigned DefReg = SrcReg; 656 if (DstTy != LLT::scalar(8)) { 657 DefReg = MRI.createVirtualRegister(getRegClass(DstTy, DstReg, MRI)); 658 BuildMI(*I.getParent(), I, I.getDebugLoc(), 659 TII.get(TargetOpcode::SUBREG_TO_REG), DefReg) 660 .addImm(0) 661 .addReg(SrcReg) 662 .addImm(X86::sub_8bit); 663 } 664 665 MachineInstr &AndInst = 666 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AndOpc), DstReg) 667 .addReg(DefReg) 668 .addImm(1); 669 670 constrainSelectedInstRegOperands(AndInst, TII, TRI, RBI); 671 672 I.eraseFromParent(); 673 return true; 674 } 675 676 bool X86InstructionSelector::selectCmp(MachineInstr &I, 677 MachineRegisterInfo &MRI, 678 MachineFunction &MF) const { 679 if (I.getOpcode() != TargetOpcode::G_ICMP) 680 return false; 681 682 X86::CondCode CC; 683 bool SwapArgs; 684 std::tie(CC, SwapArgs) = X86::getX86ConditionCode( 685 (CmpInst::Predicate)I.getOperand(1).getPredicate()); 686 unsigned OpSet = X86::getSETFromCond(CC); 687 688 unsigned LHS = I.getOperand(2).getReg(); 689 unsigned RHS = I.getOperand(3).getReg(); 690 691 if (SwapArgs) 692 std::swap(LHS, RHS); 693 694 unsigned OpCmp; 695 LLT Ty = MRI.getType(LHS); 696 697 switch (Ty.getSizeInBits()) { 698 default: 699 return false; 700 case 8: 701 OpCmp = X86::CMP8rr; 702 break; 703 case 16: 704 OpCmp = X86::CMP16rr; 705 break; 706 case 32: 707 OpCmp = X86::CMP32rr; 708 break; 709 case 64: 710 OpCmp = X86::CMP64rr; 711 break; 712 } 713 714 MachineInstr &CmpInst = 715 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp)) 716 .addReg(LHS) 717 .addReg(RHS); 718 719 MachineInstr &SetInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(), 720 TII.get(OpSet), I.getOperand(0).getReg()); 721 722 constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI); 723 constrainSelectedInstRegOperands(SetInst, TII, TRI, RBI); 724 725 I.eraseFromParent(); 726 return true; 727 } 728 729 bool X86InstructionSelector::selectUadde(MachineInstr &I, 730 MachineRegisterInfo &MRI, 731 MachineFunction &MF) const { 732 if (I.getOpcode() != TargetOpcode::G_UADDE) 733 return false; 734 735 const unsigned DstReg = I.getOperand(0).getReg(); 736 const unsigned CarryOutReg = I.getOperand(1).getReg(); 737 const unsigned Op0Reg = I.getOperand(2).getReg(); 738 const unsigned Op1Reg = I.getOperand(3).getReg(); 739 unsigned CarryInReg = I.getOperand(4).getReg(); 740 741 const LLT DstTy = MRI.getType(DstReg); 742 743 if (DstTy != LLT::scalar(32)) 744 return false; 745 746 // find CarryIn def instruction. 747 MachineInstr *Def = MRI.getVRegDef(CarryInReg); 748 while (Def->getOpcode() == TargetOpcode::G_TRUNC) { 749 CarryInReg = Def->getOperand(1).getReg(); 750 Def = MRI.getVRegDef(CarryInReg); 751 } 752 753 unsigned Opcode; 754 if (Def->getOpcode() == TargetOpcode::G_UADDE) { 755 // carry set by prev ADD. 756 757 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), X86::EFLAGS) 758 .addReg(CarryInReg); 759 760 if (!RBI.constrainGenericRegister(CarryInReg, X86::GR32RegClass, MRI)) 761 return false; 762 763 Opcode = X86::ADC32rr; 764 } else if (auto val = getConstantVRegVal(CarryInReg, MRI)) { 765 // carry is constant, support only 0. 766 if (*val != 0) 767 return false; 768 769 Opcode = X86::ADD32rr; 770 } else 771 return false; 772 773 MachineInstr &AddInst = 774 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode), DstReg) 775 .addReg(Op0Reg) 776 .addReg(Op1Reg); 777 778 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), CarryOutReg) 779 .addReg(X86::EFLAGS); 780 781 if (!constrainSelectedInstRegOperands(AddInst, TII, TRI, RBI) || 782 !RBI.constrainGenericRegister(CarryOutReg, X86::GR32RegClass, MRI)) 783 return false; 784 785 I.eraseFromParent(); 786 return true; 787 } 788 789 bool X86InstructionSelector::selectExtract(MachineInstr &I, 790 MachineRegisterInfo &MRI, 791 MachineFunction &MF) const { 792 793 if (I.getOpcode() != TargetOpcode::G_EXTRACT) 794 return false; 795 796 const unsigned DstReg = I.getOperand(0).getReg(); 797 const unsigned SrcReg = I.getOperand(1).getReg(); 798 int64_t Index = I.getOperand(2).getImm(); 799 800 const LLT DstTy = MRI.getType(DstReg); 801 const LLT SrcTy = MRI.getType(SrcReg); 802 803 // Meanwile handle vector type only. 804 if (!DstTy.isVector()) 805 return false; 806 807 if (Index % DstTy.getSizeInBits() != 0) 808 return false; // Not extract subvector. 809 810 if (Index == 0) { 811 // Replace by extract subreg copy. 812 if (!emitExtractSubreg(DstReg, SrcReg, I, MRI, MF)) 813 return false; 814 815 I.eraseFromParent(); 816 return true; 817 } 818 819 bool HasAVX = STI.hasAVX(); 820 bool HasAVX512 = STI.hasAVX512(); 821 bool HasVLX = STI.hasVLX(); 822 823 if (SrcTy.getSizeInBits() == 256 && DstTy.getSizeInBits() == 128) { 824 if (HasVLX) 825 I.setDesc(TII.get(X86::VEXTRACTF32x4Z256rr)); 826 else if (HasAVX) 827 I.setDesc(TII.get(X86::VEXTRACTF128rr)); 828 else 829 return false; 830 } else if (SrcTy.getSizeInBits() == 512 && HasAVX512) { 831 if (DstTy.getSizeInBits() == 128) 832 I.setDesc(TII.get(X86::VEXTRACTF32x4Zrr)); 833 else if (DstTy.getSizeInBits() == 256) 834 I.setDesc(TII.get(X86::VEXTRACTF64x4Zrr)); 835 else 836 return false; 837 } else 838 return false; 839 840 // Convert to X86 VEXTRACT immediate. 841 Index = Index / DstTy.getSizeInBits(); 842 I.getOperand(2).setImm(Index); 843 844 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 845 } 846 847 bool X86InstructionSelector::emitExtractSubreg(unsigned DstReg, unsigned SrcReg, 848 MachineInstr &I, 849 MachineRegisterInfo &MRI, 850 MachineFunction &MF) const { 851 852 const LLT DstTy = MRI.getType(DstReg); 853 const LLT SrcTy = MRI.getType(SrcReg); 854 unsigned SubIdx = X86::NoSubRegister; 855 856 if (!DstTy.isVector() || !SrcTy.isVector()) 857 return false; 858 859 assert(SrcTy.getSizeInBits() > DstTy.getSizeInBits() && 860 "Incorrect Src/Dst register size"); 861 862 if (DstTy.getSizeInBits() == 128) 863 SubIdx = X86::sub_xmm; 864 else if (DstTy.getSizeInBits() == 256) 865 SubIdx = X86::sub_ymm; 866 else 867 return false; 868 869 const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI); 870 const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI); 871 872 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx); 873 874 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) || 875 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { 876 DEBUG(dbgs() << "Failed to constrain G_TRUNC\n"); 877 return false; 878 } 879 880 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), DstReg) 881 .addReg(SrcReg, 0, SubIdx); 882 883 return true; 884 } 885 886 bool X86InstructionSelector::emitInsertSubreg(unsigned DstReg, unsigned SrcReg, 887 MachineInstr &I, 888 MachineRegisterInfo &MRI, 889 MachineFunction &MF) const { 890 891 const LLT DstTy = MRI.getType(DstReg); 892 const LLT SrcTy = MRI.getType(SrcReg); 893 unsigned SubIdx = X86::NoSubRegister; 894 895 // TODO: support scalar types 896 if (!DstTy.isVector() || !SrcTy.isVector()) 897 return false; 898 899 assert(SrcTy.getSizeInBits() < DstTy.getSizeInBits() && 900 "Incorrect Src/Dst register size"); 901 902 if (SrcTy.getSizeInBits() == 128) 903 SubIdx = X86::sub_xmm; 904 else if (SrcTy.getSizeInBits() == 256) 905 SubIdx = X86::sub_ymm; 906 else 907 return false; 908 909 const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI); 910 const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI); 911 912 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) || 913 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { 914 DEBUG(dbgs() << "Failed to constrain INSERT_SUBREG\n"); 915 return false; 916 } 917 918 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY)) 919 .addReg(DstReg, RegState::DefineNoRead, SubIdx) 920 .addReg(SrcReg); 921 922 return true; 923 } 924 925 bool X86InstructionSelector::selectInsert(MachineInstr &I, 926 MachineRegisterInfo &MRI, 927 MachineFunction &MF) const { 928 929 if (I.getOpcode() != TargetOpcode::G_INSERT) 930 return false; 931 932 const unsigned DstReg = I.getOperand(0).getReg(); 933 const unsigned SrcReg = I.getOperand(1).getReg(); 934 const unsigned InsertReg = I.getOperand(2).getReg(); 935 int64_t Index = I.getOperand(3).getImm(); 936 937 const LLT DstTy = MRI.getType(DstReg); 938 const LLT InsertRegTy = MRI.getType(InsertReg); 939 940 // Meanwile handle vector type only. 941 if (!DstTy.isVector()) 942 return false; 943 944 if (Index % InsertRegTy.getSizeInBits() != 0) 945 return false; // Not insert subvector. 946 947 if (Index == 0 && MRI.getVRegDef(SrcReg)->isImplicitDef()) { 948 // Replace by subreg copy. 949 if (!emitInsertSubreg(DstReg, InsertReg, I, MRI, MF)) 950 return false; 951 952 I.eraseFromParent(); 953 return true; 954 } 955 956 bool HasAVX = STI.hasAVX(); 957 bool HasAVX512 = STI.hasAVX512(); 958 bool HasVLX = STI.hasVLX(); 959 960 if (DstTy.getSizeInBits() == 256 && InsertRegTy.getSizeInBits() == 128) { 961 if (HasVLX) 962 I.setDesc(TII.get(X86::VINSERTF32x4Z256rr)); 963 else if (HasAVX) 964 I.setDesc(TII.get(X86::VINSERTF128rr)); 965 else 966 return false; 967 } else if (DstTy.getSizeInBits() == 512 && HasAVX512) { 968 if (InsertRegTy.getSizeInBits() == 128) 969 I.setDesc(TII.get(X86::VINSERTF32x4Zrr)); 970 else if (InsertRegTy.getSizeInBits() == 256) 971 I.setDesc(TII.get(X86::VINSERTF64x4Zrr)); 972 else 973 return false; 974 } else 975 return false; 976 977 // Convert to X86 VINSERT immediate. 978 Index = Index / InsertRegTy.getSizeInBits(); 979 980 I.getOperand(3).setImm(Index); 981 982 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 983 } 984 985 bool X86InstructionSelector::selectUnmergeValues(MachineInstr &I, 986 MachineRegisterInfo &MRI, 987 MachineFunction &MF) const { 988 if (I.getOpcode() != TargetOpcode::G_UNMERGE_VALUES) 989 return false; 990 991 // Split to extracts. 992 unsigned NumDefs = I.getNumOperands() - 1; 993 unsigned SrcReg = I.getOperand(NumDefs).getReg(); 994 unsigned DefSize = MRI.getType(I.getOperand(0).getReg()).getSizeInBits(); 995 996 for (unsigned Idx = 0; Idx < NumDefs; ++Idx) { 997 998 MachineInstr &ExtrInst = 999 *BuildMI(*I.getParent(), I, I.getDebugLoc(), 1000 TII.get(TargetOpcode::G_EXTRACT), I.getOperand(Idx).getReg()) 1001 .addReg(SrcReg) 1002 .addImm(Idx * DefSize); 1003 1004 if (!select(ExtrInst)) 1005 return false; 1006 } 1007 1008 I.eraseFromParent(); 1009 return true; 1010 } 1011 1012 bool X86InstructionSelector::selectMergeValues(MachineInstr &I, 1013 MachineRegisterInfo &MRI, 1014 MachineFunction &MF) const { 1015 if (I.getOpcode() != TargetOpcode::G_MERGE_VALUES) 1016 return false; 1017 1018 // Split to inserts. 1019 unsigned DstReg = I.getOperand(0).getReg(); 1020 unsigned SrcReg0 = I.getOperand(1).getReg(); 1021 1022 const LLT DstTy = MRI.getType(DstReg); 1023 const LLT SrcTy = MRI.getType(SrcReg0); 1024 unsigned SrcSize = SrcTy.getSizeInBits(); 1025 1026 const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI); 1027 1028 // For the first src use insertSubReg. 1029 unsigned DefReg = MRI.createGenericVirtualRegister(DstTy); 1030 MRI.setRegBank(DefReg, RegBank); 1031 if (!emitInsertSubreg(DefReg, I.getOperand(1).getReg(), I, MRI, MF)) 1032 return false; 1033 1034 for (unsigned Idx = 2; Idx < I.getNumOperands(); ++Idx) { 1035 1036 unsigned Tmp = MRI.createGenericVirtualRegister(DstTy); 1037 MRI.setRegBank(Tmp, RegBank); 1038 1039 MachineInstr &InsertInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(), 1040 TII.get(TargetOpcode::G_INSERT), Tmp) 1041 .addReg(DefReg) 1042 .addReg(I.getOperand(Idx).getReg()) 1043 .addImm((Idx - 1) * SrcSize); 1044 1045 DefReg = Tmp; 1046 1047 if (!select(InsertInst)) 1048 return false; 1049 } 1050 1051 MachineInstr &CopyInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(), 1052 TII.get(TargetOpcode::COPY), DstReg) 1053 .addReg(DefReg); 1054 1055 if (!select(CopyInst)) 1056 return false; 1057 1058 I.eraseFromParent(); 1059 return true; 1060 } 1061 InstructionSelector * 1062 llvm::createX86InstructionSelector(const X86TargetMachine &TM, 1063 X86Subtarget &Subtarget, 1064 X86RegisterBankInfo &RBI) { 1065 return new X86InstructionSelector(TM, Subtarget, RBI); 1066 } 1067