1 //===- X86InstructionSelector.cpp ----------------------------*- C++ -*-==// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 /// \file 10 /// This file implements the targeting of the InstructionSelector class for 11 /// X86. 12 /// \todo This should be generated by TableGen. 13 //===----------------------------------------------------------------------===// 14 15 #include "X86InstrBuilder.h" 16 #include "X86InstrInfo.h" 17 #include "X86RegisterBankInfo.h" 18 #include "X86RegisterInfo.h" 19 #include "X86Subtarget.h" 20 #include "X86TargetMachine.h" 21 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 22 #include "llvm/CodeGen/GlobalISel/Utils.h" 23 #include "llvm/CodeGen/MachineBasicBlock.h" 24 #include "llvm/CodeGen/MachineFunction.h" 25 #include "llvm/CodeGen/MachineInstr.h" 26 #include "llvm/CodeGen/MachineInstrBuilder.h" 27 #include "llvm/CodeGen/MachineOperand.h" 28 #include "llvm/CodeGen/MachineRegisterInfo.h" 29 #include "llvm/IR/Type.h" 30 #include "llvm/Support/Debug.h" 31 #include "llvm/Support/raw_ostream.h" 32 33 #define DEBUG_TYPE "X86-isel" 34 35 using namespace llvm; 36 37 #ifndef LLVM_BUILD_GLOBAL_ISEL 38 #error "You shouldn't build this" 39 #endif 40 41 namespace { 42 43 #define GET_GLOBALISEL_PREDICATE_BITSET 44 #include "X86GenGlobalISel.inc" 45 #undef GET_GLOBALISEL_PREDICATE_BITSET 46 47 class X86InstructionSelector : public InstructionSelector { 48 public: 49 X86InstructionSelector(const X86TargetMachine &TM, const X86Subtarget &STI, 50 const X86RegisterBankInfo &RBI); 51 52 bool select(MachineInstr &I) const override; 53 54 private: 55 /// tblgen-erated 'select' implementation, used as the initial selector for 56 /// the patterns that don't require complex C++. 57 bool selectImpl(MachineInstr &I) const; 58 59 // TODO: remove after suported by Tablegen-erated instruction selection. 60 unsigned getLoadStoreOp(LLT &Ty, const RegisterBank &RB, unsigned Opc, 61 uint64_t Alignment) const; 62 63 bool selectLoadStoreOp(MachineInstr &I, MachineRegisterInfo &MRI, 64 MachineFunction &MF) const; 65 bool selectFrameIndexOrGep(MachineInstr &I, MachineRegisterInfo &MRI, 66 MachineFunction &MF) const; 67 bool selectConstant(MachineInstr &I, MachineRegisterInfo &MRI, 68 MachineFunction &MF) const; 69 bool selectTrunc(MachineInstr &I, MachineRegisterInfo &MRI, 70 MachineFunction &MF) const; 71 bool selectZext(MachineInstr &I, MachineRegisterInfo &MRI, 72 MachineFunction &MF) const; 73 bool selectCmp(MachineInstr &I, MachineRegisterInfo &MRI, 74 MachineFunction &MF) const; 75 bool selectUadde(MachineInstr &I, MachineRegisterInfo &MRI, 76 MachineFunction &MF) const; 77 bool selectCopy(MachineInstr &I, MachineRegisterInfo &MRI) const; 78 bool selectInsert(MachineInstr &I, MachineRegisterInfo &MRI, 79 MachineFunction &MF) const; 80 bool selectExtract(MachineInstr &I, MachineRegisterInfo &MRI, 81 MachineFunction &MF) const; 82 83 // emit insert subreg instruction and insert it before MachineInstr &I 84 bool emitInsertSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I, 85 MachineRegisterInfo &MRI, MachineFunction &MF) const; 86 // emit extract subreg instruction and insert it before MachineInstr &I 87 bool emitExtractSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I, 88 MachineRegisterInfo &MRI, MachineFunction &MF) const; 89 90 const TargetRegisterClass *getRegClass(LLT Ty, const RegisterBank &RB) const; 91 const TargetRegisterClass *getRegClass(LLT Ty, unsigned Reg, 92 MachineRegisterInfo &MRI) const; 93 94 const X86TargetMachine &TM; 95 const X86Subtarget &STI; 96 const X86InstrInfo &TII; 97 const X86RegisterInfo &TRI; 98 const X86RegisterBankInfo &RBI; 99 100 #define GET_GLOBALISEL_PREDICATES_DECL 101 #include "X86GenGlobalISel.inc" 102 #undef GET_GLOBALISEL_PREDICATES_DECL 103 104 #define GET_GLOBALISEL_TEMPORARIES_DECL 105 #include "X86GenGlobalISel.inc" 106 #undef GET_GLOBALISEL_TEMPORARIES_DECL 107 }; 108 109 } // end anonymous namespace 110 111 #define GET_GLOBALISEL_IMPL 112 #include "X86GenGlobalISel.inc" 113 #undef GET_GLOBALISEL_IMPL 114 115 X86InstructionSelector::X86InstructionSelector(const X86TargetMachine &TM, 116 const X86Subtarget &STI, 117 const X86RegisterBankInfo &RBI) 118 : InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()), 119 TRI(*STI.getRegisterInfo()), RBI(RBI), 120 #define GET_GLOBALISEL_PREDICATES_INIT 121 #include "X86GenGlobalISel.inc" 122 #undef GET_GLOBALISEL_PREDICATES_INIT 123 #define GET_GLOBALISEL_TEMPORARIES_INIT 124 #include "X86GenGlobalISel.inc" 125 #undef GET_GLOBALISEL_TEMPORARIES_INIT 126 { 127 } 128 129 // FIXME: This should be target-independent, inferred from the types declared 130 // for each class in the bank. 131 const TargetRegisterClass * 132 X86InstructionSelector::getRegClass(LLT Ty, const RegisterBank &RB) const { 133 if (RB.getID() == X86::GPRRegBankID) { 134 if (Ty.getSizeInBits() <= 8) 135 return &X86::GR8RegClass; 136 if (Ty.getSizeInBits() == 16) 137 return &X86::GR16RegClass; 138 if (Ty.getSizeInBits() == 32) 139 return &X86::GR32RegClass; 140 if (Ty.getSizeInBits() == 64) 141 return &X86::GR64RegClass; 142 } 143 if (RB.getID() == X86::VECRRegBankID) { 144 if (Ty.getSizeInBits() == 32) 145 return STI.hasAVX512() ? &X86::FR32XRegClass : &X86::FR32RegClass; 146 if (Ty.getSizeInBits() == 64) 147 return STI.hasAVX512() ? &X86::FR64XRegClass : &X86::FR64RegClass; 148 if (Ty.getSizeInBits() == 128) 149 return STI.hasAVX512() ? &X86::VR128XRegClass : &X86::VR128RegClass; 150 if (Ty.getSizeInBits() == 256) 151 return STI.hasAVX512() ? &X86::VR256XRegClass : &X86::VR256RegClass; 152 if (Ty.getSizeInBits() == 512) 153 return &X86::VR512RegClass; 154 } 155 156 llvm_unreachable("Unknown RegBank!"); 157 } 158 159 const TargetRegisterClass * 160 X86InstructionSelector::getRegClass(LLT Ty, unsigned Reg, 161 MachineRegisterInfo &MRI) const { 162 const RegisterBank &RegBank = *RBI.getRegBank(Reg, MRI, TRI); 163 return getRegClass(Ty, RegBank); 164 } 165 166 // Set X86 Opcode and constrain DestReg. 167 bool X86InstructionSelector::selectCopy(MachineInstr &I, 168 MachineRegisterInfo &MRI) const { 169 170 unsigned DstReg = I.getOperand(0).getReg(); 171 if (TargetRegisterInfo::isPhysicalRegister(DstReg)) { 172 assert(I.isCopy() && "Generic operators do not allow physical registers"); 173 return true; 174 } 175 176 const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI); 177 const unsigned DstSize = MRI.getType(DstReg).getSizeInBits(); 178 unsigned SrcReg = I.getOperand(1).getReg(); 179 const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI); 180 181 assert((!TargetRegisterInfo::isPhysicalRegister(SrcReg) || I.isCopy()) && 182 "No phys reg on generic operators"); 183 assert((DstSize == SrcSize || 184 // Copies are a mean to setup initial types, the number of 185 // bits may not exactly match. 186 (TargetRegisterInfo::isPhysicalRegister(SrcReg) && 187 DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI))) && 188 "Copy with different width?!"); 189 190 const TargetRegisterClass *RC = nullptr; 191 192 switch (RegBank.getID()) { 193 case X86::GPRRegBankID: 194 assert((DstSize <= 64) && "GPRs cannot get more than 64-bit width values."); 195 RC = getRegClass(MRI.getType(DstReg), RegBank); 196 197 // Change the physical register 198 if (SrcSize > DstSize && TargetRegisterInfo::isPhysicalRegister(SrcReg)) { 199 if (RC == &X86::GR32RegClass) 200 I.getOperand(1).setSubReg(X86::sub_32bit); 201 else if (RC == &X86::GR16RegClass) 202 I.getOperand(1).setSubReg(X86::sub_16bit); 203 else if (RC == &X86::GR8RegClass) 204 I.getOperand(1).setSubReg(X86::sub_8bit); 205 206 I.getOperand(1).substPhysReg(SrcReg, TRI); 207 } 208 break; 209 case X86::VECRRegBankID: 210 RC = getRegClass(MRI.getType(DstReg), RegBank); 211 break; 212 default: 213 llvm_unreachable("Unknown RegBank!"); 214 } 215 216 // No need to constrain SrcReg. It will get constrained when 217 // we hit another of its use or its defs. 218 // Copies do not have constraints. 219 const TargetRegisterClass *OldRC = MRI.getRegClassOrNull(DstReg); 220 if (!OldRC || !RC->hasSubClassEq(OldRC)) { 221 if (!RBI.constrainGenericRegister(DstReg, *RC, MRI)) { 222 DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode()) 223 << " operand\n"); 224 return false; 225 } 226 } 227 I.setDesc(TII.get(X86::COPY)); 228 return true; 229 } 230 231 bool X86InstructionSelector::select(MachineInstr &I) const { 232 assert(I.getParent() && "Instruction should be in a basic block!"); 233 assert(I.getParent()->getParent() && "Instruction should be in a function!"); 234 235 MachineBasicBlock &MBB = *I.getParent(); 236 MachineFunction &MF = *MBB.getParent(); 237 MachineRegisterInfo &MRI = MF.getRegInfo(); 238 239 unsigned Opcode = I.getOpcode(); 240 if (!isPreISelGenericOpcode(Opcode)) { 241 // Certain non-generic instructions also need some special handling. 242 243 if (I.isCopy()) 244 return selectCopy(I, MRI); 245 246 // TODO: handle more cases - LOAD_STACK_GUARD, PHI 247 return true; 248 } 249 250 assert(I.getNumOperands() == I.getNumExplicitOperands() && 251 "Generic instruction has unexpected implicit operands\n"); 252 253 if (selectImpl(I)) 254 return true; 255 256 DEBUG(dbgs() << " C++ instruction selection: "; I.print(dbgs())); 257 258 // TODO: This should be implemented by tblgen. 259 if (selectLoadStoreOp(I, MRI, MF)) 260 return true; 261 if (selectFrameIndexOrGep(I, MRI, MF)) 262 return true; 263 if (selectConstant(I, MRI, MF)) 264 return true; 265 if (selectTrunc(I, MRI, MF)) 266 return true; 267 if (selectZext(I, MRI, MF)) 268 return true; 269 if (selectCmp(I, MRI, MF)) 270 return true; 271 if (selectUadde(I, MRI, MF)) 272 return true; 273 if (selectExtract(I, MRI, MF)) 274 return true; 275 if (selectInsert(I, MRI, MF)) 276 return true; 277 278 return false; 279 } 280 281 unsigned X86InstructionSelector::getLoadStoreOp(LLT &Ty, const RegisterBank &RB, 282 unsigned Opc, 283 uint64_t Alignment) const { 284 bool Isload = (Opc == TargetOpcode::G_LOAD); 285 bool HasAVX = STI.hasAVX(); 286 bool HasAVX512 = STI.hasAVX512(); 287 bool HasVLX = STI.hasVLX(); 288 289 if (Ty == LLT::scalar(8)) { 290 if (X86::GPRRegBankID == RB.getID()) 291 return Isload ? X86::MOV8rm : X86::MOV8mr; 292 } else if (Ty == LLT::scalar(16)) { 293 if (X86::GPRRegBankID == RB.getID()) 294 return Isload ? X86::MOV16rm : X86::MOV16mr; 295 } else if (Ty == LLT::scalar(32) || Ty == LLT::pointer(0, 32)) { 296 if (X86::GPRRegBankID == RB.getID()) 297 return Isload ? X86::MOV32rm : X86::MOV32mr; 298 if (X86::VECRRegBankID == RB.getID()) 299 return Isload ? (HasAVX512 ? X86::VMOVSSZrm 300 : HasAVX ? X86::VMOVSSrm : X86::MOVSSrm) 301 : (HasAVX512 ? X86::VMOVSSZmr 302 : HasAVX ? X86::VMOVSSmr : X86::MOVSSmr); 303 } else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64)) { 304 if (X86::GPRRegBankID == RB.getID()) 305 return Isload ? X86::MOV64rm : X86::MOV64mr; 306 if (X86::VECRRegBankID == RB.getID()) 307 return Isload ? (HasAVX512 ? X86::VMOVSDZrm 308 : HasAVX ? X86::VMOVSDrm : X86::MOVSDrm) 309 : (HasAVX512 ? X86::VMOVSDZmr 310 : HasAVX ? X86::VMOVSDmr : X86::MOVSDmr); 311 } else if (Ty.isVector() && Ty.getSizeInBits() == 128) { 312 if (Alignment >= 16) 313 return Isload ? (HasVLX ? X86::VMOVAPSZ128rm 314 : HasAVX512 315 ? X86::VMOVAPSZ128rm_NOVLX 316 : HasAVX ? X86::VMOVAPSrm : X86::MOVAPSrm) 317 : (HasVLX ? X86::VMOVAPSZ128mr 318 : HasAVX512 319 ? X86::VMOVAPSZ128mr_NOVLX 320 : HasAVX ? X86::VMOVAPSmr : X86::MOVAPSmr); 321 else 322 return Isload ? (HasVLX ? X86::VMOVUPSZ128rm 323 : HasAVX512 324 ? X86::VMOVUPSZ128rm_NOVLX 325 : HasAVX ? X86::VMOVUPSrm : X86::MOVUPSrm) 326 : (HasVLX ? X86::VMOVUPSZ128mr 327 : HasAVX512 328 ? X86::VMOVUPSZ128mr_NOVLX 329 : HasAVX ? X86::VMOVUPSmr : X86::MOVUPSmr); 330 } else if (Ty.isVector() && Ty.getSizeInBits() == 256) { 331 if (Alignment >= 32) 332 return Isload ? (HasVLX ? X86::VMOVAPSZ256rm 333 : HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX 334 : X86::VMOVAPSYrm) 335 : (HasVLX ? X86::VMOVAPSZ256mr 336 : HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX 337 : X86::VMOVAPSYmr); 338 else 339 return Isload ? (HasVLX ? X86::VMOVUPSZ256rm 340 : HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX 341 : X86::VMOVUPSYrm) 342 : (HasVLX ? X86::VMOVUPSZ256mr 343 : HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX 344 : X86::VMOVUPSYmr); 345 } else if (Ty.isVector() && Ty.getSizeInBits() == 512) { 346 if (Alignment >= 64) 347 return Isload ? X86::VMOVAPSZrm : X86::VMOVAPSZmr; 348 else 349 return Isload ? X86::VMOVUPSZrm : X86::VMOVUPSZmr; 350 } 351 return Opc; 352 } 353 354 // Fill in an address from the given instruction. 355 void X86SelectAddress(const MachineInstr &I, const MachineRegisterInfo &MRI, 356 X86AddressMode &AM) { 357 358 assert(I.getOperand(0).isReg() && "unsupported opperand."); 359 assert(MRI.getType(I.getOperand(0).getReg()).isPointer() && 360 "unsupported type."); 361 362 if (I.getOpcode() == TargetOpcode::G_GEP) { 363 if (auto COff = getConstantVRegVal(I.getOperand(2).getReg(), MRI)) { 364 int64_t Imm = *COff; 365 if (isInt<32>(Imm)) { // Check for displacement overflow. 366 AM.Disp = static_cast<int32_t>(Imm); 367 AM.Base.Reg = I.getOperand(1).getReg(); 368 return; 369 } 370 } 371 } else if (I.getOpcode() == TargetOpcode::G_FRAME_INDEX) { 372 AM.Base.FrameIndex = I.getOperand(1).getIndex(); 373 AM.BaseType = X86AddressMode::FrameIndexBase; 374 return; 375 } 376 377 // Default behavior. 378 AM.Base.Reg = I.getOperand(0).getReg(); 379 return; 380 } 381 382 bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I, 383 MachineRegisterInfo &MRI, 384 MachineFunction &MF) const { 385 386 unsigned Opc = I.getOpcode(); 387 388 if (Opc != TargetOpcode::G_STORE && Opc != TargetOpcode::G_LOAD) 389 return false; 390 391 const unsigned DefReg = I.getOperand(0).getReg(); 392 LLT Ty = MRI.getType(DefReg); 393 const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI); 394 395 auto &MemOp = **I.memoperands_begin(); 396 if (MemOp.getOrdering() != AtomicOrdering::NotAtomic) { 397 DEBUG(dbgs() << "Atomic load/store not supported yet\n"); 398 return false; 399 } 400 401 unsigned NewOpc = getLoadStoreOp(Ty, RB, Opc, MemOp.getAlignment()); 402 if (NewOpc == Opc) 403 return false; 404 405 X86AddressMode AM; 406 X86SelectAddress(*MRI.getVRegDef(I.getOperand(1).getReg()), MRI, AM); 407 408 I.setDesc(TII.get(NewOpc)); 409 MachineInstrBuilder MIB(MF, I); 410 if (Opc == TargetOpcode::G_LOAD) { 411 I.RemoveOperand(1); 412 addFullAddress(MIB, AM); 413 } else { 414 // G_STORE (VAL, Addr), X86Store instruction (Addr, VAL) 415 I.RemoveOperand(1); 416 I.RemoveOperand(0); 417 addFullAddress(MIB, AM).addUse(DefReg); 418 } 419 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 420 } 421 422 bool X86InstructionSelector::selectFrameIndexOrGep(MachineInstr &I, 423 MachineRegisterInfo &MRI, 424 MachineFunction &MF) const { 425 unsigned Opc = I.getOpcode(); 426 427 if (Opc != TargetOpcode::G_FRAME_INDEX && Opc != TargetOpcode::G_GEP) 428 return false; 429 430 const unsigned DefReg = I.getOperand(0).getReg(); 431 LLT Ty = MRI.getType(DefReg); 432 433 // Use LEA to calculate frame index and GEP 434 unsigned NewOpc; 435 if (Ty == LLT::pointer(0, 64)) 436 NewOpc = X86::LEA64r; 437 else if (Ty == LLT::pointer(0, 32)) 438 NewOpc = STI.isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r; 439 else 440 llvm_unreachable("Can't select G_FRAME_INDEX/G_GEP, unsupported type."); 441 442 I.setDesc(TII.get(NewOpc)); 443 MachineInstrBuilder MIB(MF, I); 444 445 if (Opc == TargetOpcode::G_FRAME_INDEX) { 446 addOffset(MIB, 0); 447 } else { 448 MachineOperand &InxOp = I.getOperand(2); 449 I.addOperand(InxOp); // set IndexReg 450 InxOp.ChangeToImmediate(1); // set Scale 451 MIB.addImm(0).addReg(0); 452 } 453 454 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 455 } 456 457 bool X86InstructionSelector::selectConstant(MachineInstr &I, 458 MachineRegisterInfo &MRI, 459 MachineFunction &MF) const { 460 if (I.getOpcode() != TargetOpcode::G_CONSTANT) 461 return false; 462 463 const unsigned DefReg = I.getOperand(0).getReg(); 464 LLT Ty = MRI.getType(DefReg); 465 466 assert(Ty.isScalar() && "invalid element type."); 467 468 uint64_t Val = 0; 469 if (I.getOperand(1).isCImm()) { 470 Val = I.getOperand(1).getCImm()->getZExtValue(); 471 I.getOperand(1).ChangeToImmediate(Val); 472 } else if (I.getOperand(1).isImm()) { 473 Val = I.getOperand(1).getImm(); 474 } else 475 llvm_unreachable("Unsupported operand type."); 476 477 unsigned NewOpc; 478 switch (Ty.getSizeInBits()) { 479 case 8: 480 NewOpc = X86::MOV8ri; 481 break; 482 case 16: 483 NewOpc = X86::MOV16ri; 484 break; 485 case 32: 486 NewOpc = X86::MOV32ri; 487 break; 488 case 64: { 489 // TODO: in case isUInt<32>(Val), X86::MOV32ri can be used 490 if (isInt<32>(Val)) 491 NewOpc = X86::MOV64ri32; 492 else 493 NewOpc = X86::MOV64ri; 494 break; 495 } 496 default: 497 llvm_unreachable("Can't select G_CONSTANT, unsupported type."); 498 } 499 500 I.setDesc(TII.get(NewOpc)); 501 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 502 } 503 504 bool X86InstructionSelector::selectTrunc(MachineInstr &I, 505 MachineRegisterInfo &MRI, 506 MachineFunction &MF) const { 507 if (I.getOpcode() != TargetOpcode::G_TRUNC) 508 return false; 509 510 const unsigned DstReg = I.getOperand(0).getReg(); 511 const unsigned SrcReg = I.getOperand(1).getReg(); 512 513 const LLT DstTy = MRI.getType(DstReg); 514 const LLT SrcTy = MRI.getType(SrcReg); 515 516 const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI); 517 const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI); 518 519 if (DstRB.getID() != SrcRB.getID()) { 520 DEBUG(dbgs() << "G_TRUNC input/output on different banks\n"); 521 return false; 522 } 523 524 if (DstRB.getID() != X86::GPRRegBankID) 525 return false; 526 527 const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB); 528 if (!DstRC) 529 return false; 530 531 const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB); 532 if (!SrcRC) 533 return false; 534 535 unsigned SubIdx; 536 if (DstRC == SrcRC) { 537 // Nothing to be done 538 SubIdx = X86::NoSubRegister; 539 } else if (DstRC == &X86::GR32RegClass) { 540 SubIdx = X86::sub_32bit; 541 } else if (DstRC == &X86::GR16RegClass) { 542 SubIdx = X86::sub_16bit; 543 } else if (DstRC == &X86::GR8RegClass) { 544 SubIdx = X86::sub_8bit; 545 } else { 546 return false; 547 } 548 549 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx); 550 551 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) || 552 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { 553 DEBUG(dbgs() << "Failed to constrain G_TRUNC\n"); 554 return false; 555 } 556 557 I.getOperand(1).setSubReg(SubIdx); 558 559 I.setDesc(TII.get(X86::COPY)); 560 return true; 561 } 562 563 bool X86InstructionSelector::selectZext(MachineInstr &I, 564 MachineRegisterInfo &MRI, 565 MachineFunction &MF) const { 566 if (I.getOpcode() != TargetOpcode::G_ZEXT) 567 return false; 568 569 const unsigned DstReg = I.getOperand(0).getReg(); 570 const unsigned SrcReg = I.getOperand(1).getReg(); 571 572 const LLT DstTy = MRI.getType(DstReg); 573 const LLT SrcTy = MRI.getType(SrcReg); 574 575 if (SrcTy == LLT::scalar(1)) { 576 577 unsigned AndOpc; 578 if (DstTy == LLT::scalar(32)) 579 AndOpc = X86::AND32ri8; 580 else if (DstTy == LLT::scalar(64)) 581 AndOpc = X86::AND64ri8; 582 else 583 return false; 584 585 unsigned DefReg = 586 MRI.createVirtualRegister(getRegClass(DstTy, DstReg, MRI)); 587 588 BuildMI(*I.getParent(), I, I.getDebugLoc(), 589 TII.get(TargetOpcode::SUBREG_TO_REG), DefReg) 590 .addImm(0) 591 .addReg(SrcReg) 592 .addImm(X86::sub_8bit); 593 594 MachineInstr &AndInst = 595 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AndOpc), DstReg) 596 .addReg(DefReg) 597 .addImm(1); 598 599 constrainSelectedInstRegOperands(AndInst, TII, TRI, RBI); 600 601 I.eraseFromParent(); 602 return true; 603 } 604 605 return false; 606 } 607 608 bool X86InstructionSelector::selectCmp(MachineInstr &I, 609 MachineRegisterInfo &MRI, 610 MachineFunction &MF) const { 611 if (I.getOpcode() != TargetOpcode::G_ICMP) 612 return false; 613 614 X86::CondCode CC; 615 bool SwapArgs; 616 std::tie(CC, SwapArgs) = X86::getX86ConditionCode( 617 (CmpInst::Predicate)I.getOperand(1).getPredicate()); 618 unsigned OpSet = X86::getSETFromCond(CC); 619 620 unsigned LHS = I.getOperand(2).getReg(); 621 unsigned RHS = I.getOperand(3).getReg(); 622 623 if (SwapArgs) 624 std::swap(LHS, RHS); 625 626 unsigned OpCmp; 627 LLT Ty = MRI.getType(LHS); 628 629 switch (Ty.getSizeInBits()) { 630 default: 631 return false; 632 case 8: 633 OpCmp = X86::CMP8rr; 634 break; 635 case 16: 636 OpCmp = X86::CMP16rr; 637 break; 638 case 32: 639 OpCmp = X86::CMP32rr; 640 break; 641 case 64: 642 OpCmp = X86::CMP64rr; 643 break; 644 } 645 646 MachineInstr &CmpInst = 647 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp)) 648 .addReg(LHS) 649 .addReg(RHS); 650 651 MachineInstr &SetInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(), 652 TII.get(OpSet), I.getOperand(0).getReg()); 653 654 constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI); 655 constrainSelectedInstRegOperands(SetInst, TII, TRI, RBI); 656 657 I.eraseFromParent(); 658 return true; 659 } 660 661 bool X86InstructionSelector::selectUadde(MachineInstr &I, 662 MachineRegisterInfo &MRI, 663 MachineFunction &MF) const { 664 if (I.getOpcode() != TargetOpcode::G_UADDE) 665 return false; 666 667 const unsigned DstReg = I.getOperand(0).getReg(); 668 const unsigned CarryOutReg = I.getOperand(1).getReg(); 669 const unsigned Op0Reg = I.getOperand(2).getReg(); 670 const unsigned Op1Reg = I.getOperand(3).getReg(); 671 unsigned CarryInReg = I.getOperand(4).getReg(); 672 673 const LLT DstTy = MRI.getType(DstReg); 674 675 if (DstTy != LLT::scalar(32)) 676 return false; 677 678 // find CarryIn def instruction. 679 MachineInstr *Def = MRI.getVRegDef(CarryInReg); 680 while (Def->getOpcode() == TargetOpcode::G_TRUNC) { 681 CarryInReg = Def->getOperand(1).getReg(); 682 Def = MRI.getVRegDef(CarryInReg); 683 } 684 685 unsigned Opcode; 686 if (Def->getOpcode() == TargetOpcode::G_UADDE) { 687 // carry set by prev ADD. 688 689 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), X86::EFLAGS) 690 .addReg(CarryInReg); 691 692 if (!RBI.constrainGenericRegister(CarryInReg, X86::GR32RegClass, MRI)) 693 return false; 694 695 Opcode = X86::ADC32rr; 696 } else if (auto val = getConstantVRegVal(CarryInReg, MRI)) { 697 // carry is constant, support only 0. 698 if (*val != 0) 699 return false; 700 701 Opcode = X86::ADD32rr; 702 } else 703 return false; 704 705 MachineInstr &AddInst = 706 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode), DstReg) 707 .addReg(Op0Reg) 708 .addReg(Op1Reg); 709 710 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), CarryOutReg) 711 .addReg(X86::EFLAGS); 712 713 if (!constrainSelectedInstRegOperands(AddInst, TII, TRI, RBI) || 714 !RBI.constrainGenericRegister(CarryOutReg, X86::GR32RegClass, MRI)) 715 return false; 716 717 I.eraseFromParent(); 718 return true; 719 } 720 721 bool X86InstructionSelector::selectExtract(MachineInstr &I, 722 MachineRegisterInfo &MRI, 723 MachineFunction &MF) const { 724 725 if (I.getOpcode() != TargetOpcode::G_EXTRACT) 726 return false; 727 728 const unsigned DstReg = I.getOperand(0).getReg(); 729 const unsigned SrcReg = I.getOperand(1).getReg(); 730 int64_t Index = I.getOperand(2).getImm(); 731 732 const LLT DstTy = MRI.getType(DstReg); 733 const LLT SrcTy = MRI.getType(SrcReg); 734 735 // Meanwile handle vector type only. 736 if (!DstTy.isVector()) 737 return false; 738 739 if (Index % DstTy.getSizeInBits() != 0) 740 return false; // Not extract subvector. 741 742 if (Index == 0) { 743 // Replace by extract subreg copy. 744 if (!emitExtractSubreg(DstReg, SrcReg, I, MRI, MF)) 745 return false; 746 747 I.eraseFromParent(); 748 return true; 749 } 750 751 bool HasAVX = STI.hasAVX(); 752 bool HasAVX512 = STI.hasAVX512(); 753 bool HasVLX = STI.hasVLX(); 754 755 if (SrcTy.getSizeInBits() == 256 && DstTy.getSizeInBits() == 128) { 756 if (HasVLX) 757 I.setDesc(TII.get(X86::VEXTRACTF32x4Z256rr)); 758 else if (HasAVX) 759 I.setDesc(TII.get(X86::VEXTRACTF128rr)); 760 else 761 return false; 762 } else if (SrcTy.getSizeInBits() == 512 && HasAVX512) { 763 if (DstTy.getSizeInBits() == 128) 764 I.setDesc(TII.get(X86::VEXTRACTF32x4Zrr)); 765 else if (DstTy.getSizeInBits() == 256) 766 I.setDesc(TII.get(X86::VEXTRACTF64x4Zrr)); 767 else 768 return false; 769 } else 770 return false; 771 772 // Convert to X86 VEXTRACT immediate. 773 Index = Index / DstTy.getSizeInBits(); 774 I.getOperand(2).setImm(Index); 775 776 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 777 } 778 779 bool X86InstructionSelector::emitExtractSubreg(unsigned DstReg, unsigned SrcReg, 780 MachineInstr &I, 781 MachineRegisterInfo &MRI, 782 MachineFunction &MF) const { 783 784 const LLT DstTy = MRI.getType(DstReg); 785 const LLT SrcTy = MRI.getType(SrcReg); 786 unsigned SubIdx = X86::NoSubRegister; 787 788 if (!DstTy.isVector() || !SrcTy.isVector()) 789 return false; 790 791 assert(SrcTy.getSizeInBits() > DstTy.getSizeInBits() && 792 "Incorrect Src/Dst register size"); 793 794 if (DstTy.getSizeInBits() == 128) 795 SubIdx = X86::sub_xmm; 796 else if (DstTy.getSizeInBits() == 256) 797 SubIdx = X86::sub_ymm; 798 else 799 return false; 800 801 const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI); 802 const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI); 803 804 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx); 805 806 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) || 807 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { 808 DEBUG(dbgs() << "Failed to constrain G_TRUNC\n"); 809 return false; 810 } 811 812 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), DstReg) 813 .addReg(SrcReg, 0, SubIdx); 814 815 return true; 816 } 817 818 bool X86InstructionSelector::emitInsertSubreg(unsigned DstReg, unsigned SrcReg, 819 MachineInstr &I, 820 MachineRegisterInfo &MRI, 821 MachineFunction &MF) const { 822 823 const LLT DstTy = MRI.getType(DstReg); 824 const LLT SrcTy = MRI.getType(SrcReg); 825 unsigned SubIdx = X86::NoSubRegister; 826 827 // TODO: support scalar types 828 if (!DstTy.isVector() || !SrcTy.isVector()) 829 return false; 830 831 assert(SrcTy.getSizeInBits() < DstTy.getSizeInBits() && 832 "Incorrect Src/Dst register size"); 833 834 if (SrcTy.getSizeInBits() == 128) 835 SubIdx = X86::sub_xmm; 836 else if (SrcTy.getSizeInBits() == 256) 837 SubIdx = X86::sub_ymm; 838 else 839 return false; 840 841 const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI); 842 const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI); 843 844 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) || 845 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { 846 DEBUG(dbgs() << "Failed to constrain INSERT_SUBREG\n"); 847 return false; 848 } 849 850 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY)) 851 .addReg(DstReg, RegState::DefineNoRead, SubIdx) 852 .addReg(SrcReg); 853 854 return true; 855 } 856 857 bool X86InstructionSelector::selectInsert(MachineInstr &I, 858 MachineRegisterInfo &MRI, 859 MachineFunction &MF) const { 860 861 if (I.getOpcode() != TargetOpcode::G_INSERT) 862 return false; 863 864 const unsigned DstReg = I.getOperand(0).getReg(); 865 const unsigned SrcReg = I.getOperand(1).getReg(); 866 const unsigned InsertReg = I.getOperand(2).getReg(); 867 int64_t Index = I.getOperand(3).getImm(); 868 869 const LLT DstTy = MRI.getType(DstReg); 870 const LLT InsertRegTy = MRI.getType(InsertReg); 871 872 // Meanwile handle vector type only. 873 if (!DstTy.isVector()) 874 return false; 875 876 if (Index % InsertRegTy.getSizeInBits() != 0) 877 return false; // Not insert subvector. 878 879 if (Index == 0 && MRI.getVRegDef(SrcReg)->isImplicitDef()) { 880 // Replace by subreg copy. 881 if (!emitInsertSubreg(DstReg, InsertReg, I, MRI, MF)) 882 return false; 883 884 I.eraseFromParent(); 885 return true; 886 } 887 888 bool HasAVX = STI.hasAVX(); 889 bool HasAVX512 = STI.hasAVX512(); 890 bool HasVLX = STI.hasVLX(); 891 892 if (DstTy.getSizeInBits() == 256 && InsertRegTy.getSizeInBits() == 128) { 893 if (HasVLX) 894 I.setDesc(TII.get(X86::VINSERTF32x4Z256rr)); 895 else if (HasAVX) 896 I.setDesc(TII.get(X86::VINSERTF128rr)); 897 else 898 return false; 899 } else if (DstTy.getSizeInBits() == 512 && HasAVX512) { 900 if (InsertRegTy.getSizeInBits() == 128) 901 I.setDesc(TII.get(X86::VINSERTF32x4Zrr)); 902 else if (InsertRegTy.getSizeInBits() == 256) 903 I.setDesc(TII.get(X86::VINSERTF64x4Zrr)); 904 else 905 return false; 906 } else 907 return false; 908 909 // Convert to X86 VINSERT immediate. 910 Index = Index / InsertRegTy.getSizeInBits(); 911 912 I.getOperand(3).setImm(Index); 913 914 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 915 } 916 917 InstructionSelector * 918 llvm::createX86InstructionSelector(const X86TargetMachine &TM, 919 X86Subtarget &Subtarget, 920 X86RegisterBankInfo &RBI) { 921 return new X86InstructionSelector(TM, Subtarget, RBI); 922 } 923