1 //===-- X86InstrInfo.cpp - X86 Instruction Information --------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file contains the X86 implementation of the TargetInstrInfo class. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "X86InstrInfo.h" 15 #include "X86.h" 16 #include "X86InstrBuilder.h" 17 #include "X86InstrFoldTables.h" 18 #include "X86MachineFunctionInfo.h" 19 #include "X86Subtarget.h" 20 #include "X86TargetMachine.h" 21 #include "llvm/ADT/STLExtras.h" 22 #include "llvm/CodeGen/LivePhysRegs.h" 23 #include "llvm/CodeGen/LiveVariables.h" 24 #include "llvm/CodeGen/MachineConstantPool.h" 25 #include "llvm/CodeGen/MachineDominators.h" 26 #include "llvm/CodeGen/MachineFrameInfo.h" 27 #include "llvm/CodeGen/MachineInstrBuilder.h" 28 #include "llvm/CodeGen/MachineModuleInfo.h" 29 #include "llvm/CodeGen/MachineRegisterInfo.h" 30 #include "llvm/CodeGen/StackMaps.h" 31 #include "llvm/IR/DerivedTypes.h" 32 #include "llvm/IR/Function.h" 33 #include "llvm/IR/LLVMContext.h" 34 #include "llvm/MC/MCAsmInfo.h" 35 #include "llvm/MC/MCExpr.h" 36 #include "llvm/MC/MCInst.h" 37 #include "llvm/Support/CommandLine.h" 38 #include "llvm/Support/Debug.h" 39 #include "llvm/Support/ErrorHandling.h" 40 #include "llvm/Support/raw_ostream.h" 41 #include "llvm/Target/TargetOptions.h" 42 43 using namespace llvm; 44 45 #define DEBUG_TYPE "x86-instr-info" 46 47 #define GET_INSTRINFO_CTOR_DTOR 48 #include "X86GenInstrInfo.inc" 49 50 static cl::opt<bool> 51 NoFusing("disable-spill-fusing", 52 cl::desc("Disable fusing of spill code into instructions"), 53 cl::Hidden); 54 static cl::opt<bool> 55 PrintFailedFusing("print-failed-fuse-candidates", 56 cl::desc("Print instructions that the allocator wants to" 57 " fuse, but the X86 backend currently can't"), 58 cl::Hidden); 59 static cl::opt<bool> 60 ReMatPICStubLoad("remat-pic-stub-load", 61 cl::desc("Re-materialize load from stub in PIC mode"), 62 cl::init(false), cl::Hidden); 63 static cl::opt<unsigned> 64 PartialRegUpdateClearance("partial-reg-update-clearance", 65 cl::desc("Clearance between two register writes " 66 "for inserting XOR to avoid partial " 67 "register update"), 68 cl::init(64), cl::Hidden); 69 static cl::opt<unsigned> 70 UndefRegClearance("undef-reg-clearance", 71 cl::desc("How many idle instructions we would like before " 72 "certain undef register reads"), 73 cl::init(128), cl::Hidden); 74 75 76 // Pin the vtable to this file. 77 void X86InstrInfo::anchor() {} 78 79 X86InstrInfo::X86InstrInfo(X86Subtarget &STI) 80 : X86GenInstrInfo((STI.isTarget64BitLP64() ? X86::ADJCALLSTACKDOWN64 81 : X86::ADJCALLSTACKDOWN32), 82 (STI.isTarget64BitLP64() ? X86::ADJCALLSTACKUP64 83 : X86::ADJCALLSTACKUP32), 84 X86::CATCHRET, 85 (STI.is64Bit() ? X86::RETQ : X86::RETL)), 86 Subtarget(STI), RI(STI.getTargetTriple()) { 87 } 88 89 bool 90 X86InstrInfo::isCoalescableExtInstr(const MachineInstr &MI, 91 unsigned &SrcReg, unsigned &DstReg, 92 unsigned &SubIdx) const { 93 switch (MI.getOpcode()) { 94 default: break; 95 case X86::MOVSX16rr8: 96 case X86::MOVZX16rr8: 97 case X86::MOVSX32rr8: 98 case X86::MOVZX32rr8: 99 case X86::MOVSX64rr8: 100 if (!Subtarget.is64Bit()) 101 // It's not always legal to reference the low 8-bit of the larger 102 // register in 32-bit mode. 103 return false; 104 LLVM_FALLTHROUGH; 105 case X86::MOVSX32rr16: 106 case X86::MOVZX32rr16: 107 case X86::MOVSX64rr16: 108 case X86::MOVSX64rr32: { 109 if (MI.getOperand(0).getSubReg() || MI.getOperand(1).getSubReg()) 110 // Be conservative. 111 return false; 112 SrcReg = MI.getOperand(1).getReg(); 113 DstReg = MI.getOperand(0).getReg(); 114 switch (MI.getOpcode()) { 115 default: llvm_unreachable("Unreachable!"); 116 case X86::MOVSX16rr8: 117 case X86::MOVZX16rr8: 118 case X86::MOVSX32rr8: 119 case X86::MOVZX32rr8: 120 case X86::MOVSX64rr8: 121 SubIdx = X86::sub_8bit; 122 break; 123 case X86::MOVSX32rr16: 124 case X86::MOVZX32rr16: 125 case X86::MOVSX64rr16: 126 SubIdx = X86::sub_16bit; 127 break; 128 case X86::MOVSX64rr32: 129 SubIdx = X86::sub_32bit; 130 break; 131 } 132 return true; 133 } 134 } 135 return false; 136 } 137 138 int X86InstrInfo::getSPAdjust(const MachineInstr &MI) const { 139 const MachineFunction *MF = MI.getParent()->getParent(); 140 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering(); 141 142 if (isFrameInstr(MI)) { 143 unsigned StackAlign = TFI->getStackAlignment(); 144 int SPAdj = alignTo(getFrameSize(MI), StackAlign); 145 SPAdj -= getFrameAdjustment(MI); 146 if (!isFrameSetup(MI)) 147 SPAdj = -SPAdj; 148 return SPAdj; 149 } 150 151 // To know whether a call adjusts the stack, we need information 152 // that is bound to the following ADJCALLSTACKUP pseudo. 153 // Look for the next ADJCALLSTACKUP that follows the call. 154 if (MI.isCall()) { 155 const MachineBasicBlock *MBB = MI.getParent(); 156 auto I = ++MachineBasicBlock::const_iterator(MI); 157 for (auto E = MBB->end(); I != E; ++I) { 158 if (I->getOpcode() == getCallFrameDestroyOpcode() || 159 I->isCall()) 160 break; 161 } 162 163 // If we could not find a frame destroy opcode, then it has already 164 // been simplified, so we don't care. 165 if (I->getOpcode() != getCallFrameDestroyOpcode()) 166 return 0; 167 168 return -(I->getOperand(1).getImm()); 169 } 170 171 // Currently handle only PUSHes we can reasonably expect to see 172 // in call sequences 173 switch (MI.getOpcode()) { 174 default: 175 return 0; 176 case X86::PUSH32i8: 177 case X86::PUSH32r: 178 case X86::PUSH32rmm: 179 case X86::PUSH32rmr: 180 case X86::PUSHi32: 181 return 4; 182 case X86::PUSH64i8: 183 case X86::PUSH64r: 184 case X86::PUSH64rmm: 185 case X86::PUSH64rmr: 186 case X86::PUSH64i32: 187 return 8; 188 } 189 } 190 191 /// Return true and the FrameIndex if the specified 192 /// operand and follow operands form a reference to the stack frame. 193 bool X86InstrInfo::isFrameOperand(const MachineInstr &MI, unsigned int Op, 194 int &FrameIndex) const { 195 if (MI.getOperand(Op + X86::AddrBaseReg).isFI() && 196 MI.getOperand(Op + X86::AddrScaleAmt).isImm() && 197 MI.getOperand(Op + X86::AddrIndexReg).isReg() && 198 MI.getOperand(Op + X86::AddrDisp).isImm() && 199 MI.getOperand(Op + X86::AddrScaleAmt).getImm() == 1 && 200 MI.getOperand(Op + X86::AddrIndexReg).getReg() == 0 && 201 MI.getOperand(Op + X86::AddrDisp).getImm() == 0) { 202 FrameIndex = MI.getOperand(Op + X86::AddrBaseReg).getIndex(); 203 return true; 204 } 205 return false; 206 } 207 208 static bool isFrameLoadOpcode(int Opcode, unsigned &MemBytes) { 209 switch (Opcode) { 210 default: 211 return false; 212 case X86::MOV8rm: 213 case X86::KMOVBkm: 214 MemBytes = 1; 215 return true; 216 case X86::MOV16rm: 217 case X86::KMOVWkm: 218 MemBytes = 2; 219 return true; 220 case X86::MOV32rm: 221 case X86::MOVSSrm: 222 case X86::VMOVSSZrm: 223 case X86::VMOVSSrm: 224 case X86::KMOVDkm: 225 MemBytes = 4; 226 return true; 227 case X86::MOV64rm: 228 case X86::LD_Fp64m: 229 case X86::MOVSDrm: 230 case X86::VMOVSDrm: 231 case X86::VMOVSDZrm: 232 case X86::MMX_MOVD64rm: 233 case X86::MMX_MOVQ64rm: 234 case X86::KMOVQkm: 235 MemBytes = 8; 236 return true; 237 case X86::MOVAPSrm: 238 case X86::MOVUPSrm: 239 case X86::MOVAPDrm: 240 case X86::MOVUPDrm: 241 case X86::MOVDQArm: 242 case X86::MOVDQUrm: 243 case X86::VMOVAPSrm: 244 case X86::VMOVUPSrm: 245 case X86::VMOVAPDrm: 246 case X86::VMOVUPDrm: 247 case X86::VMOVDQArm: 248 case X86::VMOVDQUrm: 249 case X86::VMOVAPSZ128rm: 250 case X86::VMOVUPSZ128rm: 251 case X86::VMOVAPSZ128rm_NOVLX: 252 case X86::VMOVUPSZ128rm_NOVLX: 253 case X86::VMOVAPDZ128rm: 254 case X86::VMOVUPDZ128rm: 255 case X86::VMOVDQU8Z128rm: 256 case X86::VMOVDQU16Z128rm: 257 case X86::VMOVDQA32Z128rm: 258 case X86::VMOVDQU32Z128rm: 259 case X86::VMOVDQA64Z128rm: 260 case X86::VMOVDQU64Z128rm: 261 MemBytes = 16; 262 return true; 263 case X86::VMOVAPSYrm: 264 case X86::VMOVUPSYrm: 265 case X86::VMOVAPDYrm: 266 case X86::VMOVUPDYrm: 267 case X86::VMOVDQAYrm: 268 case X86::VMOVDQUYrm: 269 case X86::VMOVAPSZ256rm: 270 case X86::VMOVUPSZ256rm: 271 case X86::VMOVAPSZ256rm_NOVLX: 272 case X86::VMOVUPSZ256rm_NOVLX: 273 case X86::VMOVAPDZ256rm: 274 case X86::VMOVUPDZ256rm: 275 case X86::VMOVDQU8Z256rm: 276 case X86::VMOVDQU16Z256rm: 277 case X86::VMOVDQA32Z256rm: 278 case X86::VMOVDQU32Z256rm: 279 case X86::VMOVDQA64Z256rm: 280 case X86::VMOVDQU64Z256rm: 281 MemBytes = 32; 282 return true; 283 case X86::VMOVAPSZrm: 284 case X86::VMOVUPSZrm: 285 case X86::VMOVAPDZrm: 286 case X86::VMOVUPDZrm: 287 case X86::VMOVDQU8Zrm: 288 case X86::VMOVDQU16Zrm: 289 case X86::VMOVDQA32Zrm: 290 case X86::VMOVDQU32Zrm: 291 case X86::VMOVDQA64Zrm: 292 case X86::VMOVDQU64Zrm: 293 MemBytes = 64; 294 return true; 295 } 296 } 297 298 static bool isFrameStoreOpcode(int Opcode, unsigned &MemBytes) { 299 switch (Opcode) { 300 default: 301 return false; 302 case X86::MOV8mr: 303 case X86::KMOVBmk: 304 MemBytes = 1; 305 return true; 306 case X86::MOV16mr: 307 case X86::KMOVWmk: 308 MemBytes = 2; 309 return true; 310 case X86::MOV32mr: 311 case X86::MOVSSmr: 312 case X86::VMOVSSmr: 313 case X86::VMOVSSZmr: 314 case X86::KMOVDmk: 315 MemBytes = 4; 316 return true; 317 case X86::MOV64mr: 318 case X86::ST_FpP64m: 319 case X86::MOVSDmr: 320 case X86::VMOVSDmr: 321 case X86::VMOVSDZmr: 322 case X86::MMX_MOVD64mr: 323 case X86::MMX_MOVQ64mr: 324 case X86::MMX_MOVNTQmr: 325 case X86::KMOVQmk: 326 MemBytes = 8; 327 return true; 328 case X86::MOVAPSmr: 329 case X86::MOVUPSmr: 330 case X86::MOVAPDmr: 331 case X86::MOVUPDmr: 332 case X86::MOVDQAmr: 333 case X86::MOVDQUmr: 334 case X86::VMOVAPSmr: 335 case X86::VMOVUPSmr: 336 case X86::VMOVAPDmr: 337 case X86::VMOVUPDmr: 338 case X86::VMOVDQAmr: 339 case X86::VMOVDQUmr: 340 case X86::VMOVUPSZ128mr: 341 case X86::VMOVAPSZ128mr: 342 case X86::VMOVUPSZ128mr_NOVLX: 343 case X86::VMOVAPSZ128mr_NOVLX: 344 case X86::VMOVUPDZ128mr: 345 case X86::VMOVAPDZ128mr: 346 case X86::VMOVDQA32Z128mr: 347 case X86::VMOVDQU32Z128mr: 348 case X86::VMOVDQA64Z128mr: 349 case X86::VMOVDQU64Z128mr: 350 case X86::VMOVDQU8Z128mr: 351 case X86::VMOVDQU16Z128mr: 352 MemBytes = 16; 353 return true; 354 case X86::VMOVUPSYmr: 355 case X86::VMOVAPSYmr: 356 case X86::VMOVUPDYmr: 357 case X86::VMOVAPDYmr: 358 case X86::VMOVDQUYmr: 359 case X86::VMOVDQAYmr: 360 case X86::VMOVUPSZ256mr: 361 case X86::VMOVAPSZ256mr: 362 case X86::VMOVUPSZ256mr_NOVLX: 363 case X86::VMOVAPSZ256mr_NOVLX: 364 case X86::VMOVUPDZ256mr: 365 case X86::VMOVAPDZ256mr: 366 case X86::VMOVDQU8Z256mr: 367 case X86::VMOVDQU16Z256mr: 368 case X86::VMOVDQA32Z256mr: 369 case X86::VMOVDQU32Z256mr: 370 case X86::VMOVDQA64Z256mr: 371 case X86::VMOVDQU64Z256mr: 372 MemBytes = 32; 373 return true; 374 case X86::VMOVUPSZmr: 375 case X86::VMOVAPSZmr: 376 case X86::VMOVUPDZmr: 377 case X86::VMOVAPDZmr: 378 case X86::VMOVDQU8Zmr: 379 case X86::VMOVDQU16Zmr: 380 case X86::VMOVDQA32Zmr: 381 case X86::VMOVDQU32Zmr: 382 case X86::VMOVDQA64Zmr: 383 case X86::VMOVDQU64Zmr: 384 MemBytes = 64; 385 return true; 386 } 387 return false; 388 } 389 390 unsigned X86InstrInfo::isLoadFromStackSlot(const MachineInstr &MI, 391 int &FrameIndex) const { 392 unsigned Dummy; 393 return X86InstrInfo::isLoadFromStackSlot(MI, FrameIndex, Dummy); 394 } 395 396 unsigned X86InstrInfo::isLoadFromStackSlot(const MachineInstr &MI, 397 int &FrameIndex, 398 unsigned &MemBytes) const { 399 if (isFrameLoadOpcode(MI.getOpcode(), MemBytes)) 400 if (MI.getOperand(0).getSubReg() == 0 && isFrameOperand(MI, 1, FrameIndex)) 401 return MI.getOperand(0).getReg(); 402 return 0; 403 } 404 405 unsigned X86InstrInfo::isLoadFromStackSlotPostFE(const MachineInstr &MI, 406 int &FrameIndex) const { 407 unsigned Dummy; 408 if (isFrameLoadOpcode(MI.getOpcode(), Dummy)) { 409 unsigned Reg; 410 if ((Reg = isLoadFromStackSlot(MI, FrameIndex))) 411 return Reg; 412 // Check for post-frame index elimination operations 413 const MachineMemOperand *Dummy; 414 return hasLoadFromStackSlot(MI, Dummy, FrameIndex); 415 } 416 return 0; 417 } 418 419 unsigned X86InstrInfo::isStoreToStackSlot(const MachineInstr &MI, 420 int &FrameIndex) const { 421 unsigned Dummy; 422 return X86InstrInfo::isStoreToStackSlot(MI, FrameIndex, Dummy); 423 } 424 425 unsigned X86InstrInfo::isStoreToStackSlot(const MachineInstr &MI, 426 int &FrameIndex, 427 unsigned &MemBytes) const { 428 if (isFrameStoreOpcode(MI.getOpcode(), MemBytes)) 429 if (MI.getOperand(X86::AddrNumOperands).getSubReg() == 0 && 430 isFrameOperand(MI, 0, FrameIndex)) 431 return MI.getOperand(X86::AddrNumOperands).getReg(); 432 return 0; 433 } 434 435 unsigned X86InstrInfo::isStoreToStackSlotPostFE(const MachineInstr &MI, 436 int &FrameIndex) const { 437 unsigned Dummy; 438 if (isFrameStoreOpcode(MI.getOpcode(), Dummy)) { 439 unsigned Reg; 440 if ((Reg = isStoreToStackSlot(MI, FrameIndex))) 441 return Reg; 442 // Check for post-frame index elimination operations 443 const MachineMemOperand *Dummy; 444 return hasStoreToStackSlot(MI, Dummy, FrameIndex); 445 } 446 return 0; 447 } 448 449 /// Return true if register is PIC base; i.e.g defined by X86::MOVPC32r. 450 static bool regIsPICBase(unsigned BaseReg, const MachineRegisterInfo &MRI) { 451 // Don't waste compile time scanning use-def chains of physregs. 452 if (!TargetRegisterInfo::isVirtualRegister(BaseReg)) 453 return false; 454 bool isPICBase = false; 455 for (MachineRegisterInfo::def_instr_iterator I = MRI.def_instr_begin(BaseReg), 456 E = MRI.def_instr_end(); I != E; ++I) { 457 MachineInstr *DefMI = &*I; 458 if (DefMI->getOpcode() != X86::MOVPC32r) 459 return false; 460 assert(!isPICBase && "More than one PIC base?"); 461 isPICBase = true; 462 } 463 return isPICBase; 464 } 465 466 bool X86InstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI, 467 AliasAnalysis *AA) const { 468 switch (MI.getOpcode()) { 469 default: break; 470 case X86::MOV8rm: 471 case X86::MOV8rm_NOREX: 472 case X86::MOV16rm: 473 case X86::MOV32rm: 474 case X86::MOV64rm: 475 case X86::LD_Fp64m: 476 case X86::MOVSSrm: 477 case X86::MOVSDrm: 478 case X86::MOVAPSrm: 479 case X86::MOVUPSrm: 480 case X86::MOVAPDrm: 481 case X86::MOVUPDrm: 482 case X86::MOVDQArm: 483 case X86::MOVDQUrm: 484 case X86::VMOVSSrm: 485 case X86::VMOVSDrm: 486 case X86::VMOVAPSrm: 487 case X86::VMOVUPSrm: 488 case X86::VMOVAPDrm: 489 case X86::VMOVUPDrm: 490 case X86::VMOVDQArm: 491 case X86::VMOVDQUrm: 492 case X86::VMOVAPSYrm: 493 case X86::VMOVUPSYrm: 494 case X86::VMOVAPDYrm: 495 case X86::VMOVUPDYrm: 496 case X86::VMOVDQAYrm: 497 case X86::VMOVDQUYrm: 498 case X86::MMX_MOVD64rm: 499 case X86::MMX_MOVQ64rm: 500 // AVX-512 501 case X86::VMOVSSZrm: 502 case X86::VMOVSDZrm: 503 case X86::VMOVAPDZ128rm: 504 case X86::VMOVAPDZ256rm: 505 case X86::VMOVAPDZrm: 506 case X86::VMOVAPSZ128rm: 507 case X86::VMOVAPSZ256rm: 508 case X86::VMOVAPSZ128rm_NOVLX: 509 case X86::VMOVAPSZ256rm_NOVLX: 510 case X86::VMOVAPSZrm: 511 case X86::VMOVDQA32Z128rm: 512 case X86::VMOVDQA32Z256rm: 513 case X86::VMOVDQA32Zrm: 514 case X86::VMOVDQA64Z128rm: 515 case X86::VMOVDQA64Z256rm: 516 case X86::VMOVDQA64Zrm: 517 case X86::VMOVDQU16Z128rm: 518 case X86::VMOVDQU16Z256rm: 519 case X86::VMOVDQU16Zrm: 520 case X86::VMOVDQU32Z128rm: 521 case X86::VMOVDQU32Z256rm: 522 case X86::VMOVDQU32Zrm: 523 case X86::VMOVDQU64Z128rm: 524 case X86::VMOVDQU64Z256rm: 525 case X86::VMOVDQU64Zrm: 526 case X86::VMOVDQU8Z128rm: 527 case X86::VMOVDQU8Z256rm: 528 case X86::VMOVDQU8Zrm: 529 case X86::VMOVUPDZ128rm: 530 case X86::VMOVUPDZ256rm: 531 case X86::VMOVUPDZrm: 532 case X86::VMOVUPSZ128rm: 533 case X86::VMOVUPSZ256rm: 534 case X86::VMOVUPSZ128rm_NOVLX: 535 case X86::VMOVUPSZ256rm_NOVLX: 536 case X86::VMOVUPSZrm: { 537 // Loads from constant pools are trivially rematerializable. 538 if (MI.getOperand(1 + X86::AddrBaseReg).isReg() && 539 MI.getOperand(1 + X86::AddrScaleAmt).isImm() && 540 MI.getOperand(1 + X86::AddrIndexReg).isReg() && 541 MI.getOperand(1 + X86::AddrIndexReg).getReg() == 0 && 542 MI.isDereferenceableInvariantLoad(AA)) { 543 unsigned BaseReg = MI.getOperand(1 + X86::AddrBaseReg).getReg(); 544 if (BaseReg == 0 || BaseReg == X86::RIP) 545 return true; 546 // Allow re-materialization of PIC load. 547 if (!ReMatPICStubLoad && MI.getOperand(1 + X86::AddrDisp).isGlobal()) 548 return false; 549 const MachineFunction &MF = *MI.getParent()->getParent(); 550 const MachineRegisterInfo &MRI = MF.getRegInfo(); 551 return regIsPICBase(BaseReg, MRI); 552 } 553 return false; 554 } 555 556 case X86::LEA32r: 557 case X86::LEA64r: { 558 if (MI.getOperand(1 + X86::AddrScaleAmt).isImm() && 559 MI.getOperand(1 + X86::AddrIndexReg).isReg() && 560 MI.getOperand(1 + X86::AddrIndexReg).getReg() == 0 && 561 !MI.getOperand(1 + X86::AddrDisp).isReg()) { 562 // lea fi#, lea GV, etc. are all rematerializable. 563 if (!MI.getOperand(1 + X86::AddrBaseReg).isReg()) 564 return true; 565 unsigned BaseReg = MI.getOperand(1 + X86::AddrBaseReg).getReg(); 566 if (BaseReg == 0) 567 return true; 568 // Allow re-materialization of lea PICBase + x. 569 const MachineFunction &MF = *MI.getParent()->getParent(); 570 const MachineRegisterInfo &MRI = MF.getRegInfo(); 571 return regIsPICBase(BaseReg, MRI); 572 } 573 return false; 574 } 575 } 576 577 // All other instructions marked M_REMATERIALIZABLE are always trivially 578 // rematerializable. 579 return true; 580 } 581 582 bool X86InstrInfo::isSafeToClobberEFLAGS(MachineBasicBlock &MBB, 583 MachineBasicBlock::iterator I) const { 584 MachineBasicBlock::iterator E = MBB.end(); 585 586 // For compile time consideration, if we are not able to determine the 587 // safety after visiting 4 instructions in each direction, we will assume 588 // it's not safe. 589 MachineBasicBlock::iterator Iter = I; 590 for (unsigned i = 0; Iter != E && i < 4; ++i) { 591 bool SeenDef = false; 592 for (unsigned j = 0, e = Iter->getNumOperands(); j != e; ++j) { 593 MachineOperand &MO = Iter->getOperand(j); 594 if (MO.isRegMask() && MO.clobbersPhysReg(X86::EFLAGS)) 595 SeenDef = true; 596 if (!MO.isReg()) 597 continue; 598 if (MO.getReg() == X86::EFLAGS) { 599 if (MO.isUse()) 600 return false; 601 SeenDef = true; 602 } 603 } 604 605 if (SeenDef) 606 // This instruction defines EFLAGS, no need to look any further. 607 return true; 608 ++Iter; 609 // Skip over debug instructions. 610 while (Iter != E && Iter->isDebugInstr()) 611 ++Iter; 612 } 613 614 // It is safe to clobber EFLAGS at the end of a block of no successor has it 615 // live in. 616 if (Iter == E) { 617 for (MachineBasicBlock *S : MBB.successors()) 618 if (S->isLiveIn(X86::EFLAGS)) 619 return false; 620 return true; 621 } 622 623 MachineBasicBlock::iterator B = MBB.begin(); 624 Iter = I; 625 for (unsigned i = 0; i < 4; ++i) { 626 // If we make it to the beginning of the block, it's safe to clobber 627 // EFLAGS iff EFLAGS is not live-in. 628 if (Iter == B) 629 return !MBB.isLiveIn(X86::EFLAGS); 630 631 --Iter; 632 // Skip over debug instructions. 633 while (Iter != B && Iter->isDebugInstr()) 634 --Iter; 635 636 bool SawKill = false; 637 for (unsigned j = 0, e = Iter->getNumOperands(); j != e; ++j) { 638 MachineOperand &MO = Iter->getOperand(j); 639 // A register mask may clobber EFLAGS, but we should still look for a 640 // live EFLAGS def. 641 if (MO.isRegMask() && MO.clobbersPhysReg(X86::EFLAGS)) 642 SawKill = true; 643 if (MO.isReg() && MO.getReg() == X86::EFLAGS) { 644 if (MO.isDef()) return MO.isDead(); 645 if (MO.isKill()) SawKill = true; 646 } 647 } 648 649 if (SawKill) 650 // This instruction kills EFLAGS and doesn't redefine it, so 651 // there's no need to look further. 652 return true; 653 } 654 655 // Conservative answer. 656 return false; 657 } 658 659 void X86InstrInfo::reMaterialize(MachineBasicBlock &MBB, 660 MachineBasicBlock::iterator I, 661 unsigned DestReg, unsigned SubIdx, 662 const MachineInstr &Orig, 663 const TargetRegisterInfo &TRI) const { 664 bool ClobbersEFLAGS = false; 665 for (const MachineOperand &MO : Orig.operands()) { 666 if (MO.isReg() && MO.isDef() && MO.getReg() == X86::EFLAGS) { 667 ClobbersEFLAGS = true; 668 break; 669 } 670 } 671 672 if (ClobbersEFLAGS && !isSafeToClobberEFLAGS(MBB, I)) { 673 // The instruction clobbers EFLAGS. Re-materialize as MOV32ri to avoid side 674 // effects. 675 int Value; 676 switch (Orig.getOpcode()) { 677 case X86::MOV32r0: Value = 0; break; 678 case X86::MOV32r1: Value = 1; break; 679 case X86::MOV32r_1: Value = -1; break; 680 default: 681 llvm_unreachable("Unexpected instruction!"); 682 } 683 684 const DebugLoc &DL = Orig.getDebugLoc(); 685 BuildMI(MBB, I, DL, get(X86::MOV32ri)) 686 .add(Orig.getOperand(0)) 687 .addImm(Value); 688 } else { 689 MachineInstr *MI = MBB.getParent()->CloneMachineInstr(&Orig); 690 MBB.insert(I, MI); 691 } 692 693 MachineInstr &NewMI = *std::prev(I); 694 NewMI.substituteRegister(Orig.getOperand(0).getReg(), DestReg, SubIdx, TRI); 695 } 696 697 /// True if MI has a condition code def, e.g. EFLAGS, that is not marked dead. 698 bool X86InstrInfo::hasLiveCondCodeDef(MachineInstr &MI) const { 699 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 700 MachineOperand &MO = MI.getOperand(i); 701 if (MO.isReg() && MO.isDef() && 702 MO.getReg() == X86::EFLAGS && !MO.isDead()) { 703 return true; 704 } 705 } 706 return false; 707 } 708 709 /// Check whether the shift count for a machine operand is non-zero. 710 inline static unsigned getTruncatedShiftCount(MachineInstr &MI, 711 unsigned ShiftAmtOperandIdx) { 712 // The shift count is six bits with the REX.W prefix and five bits without. 713 unsigned ShiftCountMask = (MI.getDesc().TSFlags & X86II::REX_W) ? 63 : 31; 714 unsigned Imm = MI.getOperand(ShiftAmtOperandIdx).getImm(); 715 return Imm & ShiftCountMask; 716 } 717 718 /// Check whether the given shift count is appropriate 719 /// can be represented by a LEA instruction. 720 inline static bool isTruncatedShiftCountForLEA(unsigned ShAmt) { 721 // Left shift instructions can be transformed into load-effective-address 722 // instructions if we can encode them appropriately. 723 // A LEA instruction utilizes a SIB byte to encode its scale factor. 724 // The SIB.scale field is two bits wide which means that we can encode any 725 // shift amount less than 4. 726 return ShAmt < 4 && ShAmt > 0; 727 } 728 729 bool X86InstrInfo::classifyLEAReg(MachineInstr &MI, const MachineOperand &Src, 730 unsigned Opc, bool AllowSP, unsigned &NewSrc, 731 bool &isKill, bool &isUndef, 732 MachineOperand &ImplicitOp, 733 LiveVariables *LV) const { 734 MachineFunction &MF = *MI.getParent()->getParent(); 735 const TargetRegisterClass *RC; 736 if (AllowSP) { 737 RC = Opc != X86::LEA32r ? &X86::GR64RegClass : &X86::GR32RegClass; 738 } else { 739 RC = Opc != X86::LEA32r ? 740 &X86::GR64_NOSPRegClass : &X86::GR32_NOSPRegClass; 741 } 742 unsigned SrcReg = Src.getReg(); 743 744 // For both LEA64 and LEA32 the register already has essentially the right 745 // type (32-bit or 64-bit) we may just need to forbid SP. 746 if (Opc != X86::LEA64_32r) { 747 NewSrc = SrcReg; 748 isKill = Src.isKill(); 749 isUndef = Src.isUndef(); 750 751 if (TargetRegisterInfo::isVirtualRegister(NewSrc) && 752 !MF.getRegInfo().constrainRegClass(NewSrc, RC)) 753 return false; 754 755 return true; 756 } 757 758 // This is for an LEA64_32r and incoming registers are 32-bit. One way or 759 // another we need to add 64-bit registers to the final MI. 760 if (TargetRegisterInfo::isPhysicalRegister(SrcReg)) { 761 ImplicitOp = Src; 762 ImplicitOp.setImplicit(); 763 764 NewSrc = getX86SubSuperRegister(Src.getReg(), 64); 765 isKill = Src.isKill(); 766 isUndef = Src.isUndef(); 767 } else { 768 // Virtual register of the wrong class, we have to create a temporary 64-bit 769 // vreg to feed into the LEA. 770 NewSrc = MF.getRegInfo().createVirtualRegister(RC); 771 MachineInstr *Copy = 772 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(TargetOpcode::COPY)) 773 .addReg(NewSrc, RegState::Define | RegState::Undef, X86::sub_32bit) 774 .add(Src); 775 776 // Which is obviously going to be dead after we're done with it. 777 isKill = true; 778 isUndef = false; 779 780 if (LV) 781 LV->replaceKillInstruction(SrcReg, MI, *Copy); 782 } 783 784 // We've set all the parameters without issue. 785 return true; 786 } 787 788 /// Helper for convertToThreeAddress when 16-bit LEA is disabled, use 32-bit 789 /// LEA to form 3-address code by promoting to a 32-bit superregister and then 790 /// truncating back down to a 16-bit subregister. 791 MachineInstr *X86InstrInfo::convertToThreeAddressWithLEA( 792 unsigned MIOpc, MachineFunction::iterator &MFI, MachineInstr &MI, 793 LiveVariables *LV) const { 794 MachineBasicBlock::iterator MBBI = MI.getIterator(); 795 unsigned Dest = MI.getOperand(0).getReg(); 796 unsigned Src = MI.getOperand(1).getReg(); 797 bool isDead = MI.getOperand(0).isDead(); 798 bool isKill = MI.getOperand(1).isKill(); 799 800 MachineRegisterInfo &RegInfo = MFI->getParent()->getRegInfo(); 801 unsigned leaOutReg = RegInfo.createVirtualRegister(&X86::GR32RegClass); 802 unsigned Opc, leaInReg; 803 if (Subtarget.is64Bit()) { 804 Opc = X86::LEA64_32r; 805 leaInReg = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass); 806 } else { 807 Opc = X86::LEA32r; 808 leaInReg = RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass); 809 } 810 811 // Build and insert into an implicit UNDEF value. This is OK because 812 // well be shifting and then extracting the lower 16-bits. 813 // This has the potential to cause partial register stall. e.g. 814 // movw (%rbp,%rcx,2), %dx 815 // leal -65(%rdx), %esi 816 // But testing has shown this *does* help performance in 64-bit mode (at 817 // least on modern x86 machines). 818 BuildMI(*MFI, MBBI, MI.getDebugLoc(), get(X86::IMPLICIT_DEF), leaInReg); 819 MachineInstr *InsMI = 820 BuildMI(*MFI, MBBI, MI.getDebugLoc(), get(TargetOpcode::COPY)) 821 .addReg(leaInReg, RegState::Define, X86::sub_16bit) 822 .addReg(Src, getKillRegState(isKill)); 823 824 MachineInstrBuilder MIB = 825 BuildMI(*MFI, MBBI, MI.getDebugLoc(), get(Opc), leaOutReg); 826 switch (MIOpc) { 827 default: llvm_unreachable("Unreachable!"); 828 case X86::SHL16ri: { 829 unsigned ShAmt = MI.getOperand(2).getImm(); 830 MIB.addReg(0).addImm(1ULL << ShAmt) 831 .addReg(leaInReg, RegState::Kill).addImm(0).addReg(0); 832 break; 833 } 834 case X86::INC16r: 835 addRegOffset(MIB, leaInReg, true, 1); 836 break; 837 case X86::DEC16r: 838 addRegOffset(MIB, leaInReg, true, -1); 839 break; 840 case X86::ADD16ri: 841 case X86::ADD16ri8: 842 case X86::ADD16ri_DB: 843 case X86::ADD16ri8_DB: 844 addRegOffset(MIB, leaInReg, true, MI.getOperand(2).getImm()); 845 break; 846 case X86::ADD16rr: 847 case X86::ADD16rr_DB: { 848 unsigned Src2 = MI.getOperand(2).getReg(); 849 bool isKill2 = MI.getOperand(2).isKill(); 850 unsigned leaInReg2 = 0; 851 MachineInstr *InsMI2 = nullptr; 852 if (Src == Src2) { 853 // ADD16rr killed %reg1028, %reg1028 854 // just a single insert_subreg. 855 addRegReg(MIB, leaInReg, true, leaInReg, false); 856 } else { 857 if (Subtarget.is64Bit()) 858 leaInReg2 = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass); 859 else 860 leaInReg2 = RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass); 861 // Build and insert into an implicit UNDEF value. This is OK because 862 // well be shifting and then extracting the lower 16-bits. 863 BuildMI(*MFI, &*MIB, MI.getDebugLoc(), get(X86::IMPLICIT_DEF), leaInReg2); 864 InsMI2 = BuildMI(*MFI, &*MIB, MI.getDebugLoc(), get(TargetOpcode::COPY)) 865 .addReg(leaInReg2, RegState::Define, X86::sub_16bit) 866 .addReg(Src2, getKillRegState(isKill2)); 867 addRegReg(MIB, leaInReg, true, leaInReg2, true); 868 } 869 if (LV && isKill2 && InsMI2) 870 LV->replaceKillInstruction(Src2, MI, *InsMI2); 871 break; 872 } 873 } 874 875 MachineInstr *NewMI = MIB; 876 MachineInstr *ExtMI = 877 BuildMI(*MFI, MBBI, MI.getDebugLoc(), get(TargetOpcode::COPY)) 878 .addReg(Dest, RegState::Define | getDeadRegState(isDead)) 879 .addReg(leaOutReg, RegState::Kill, X86::sub_16bit); 880 881 if (LV) { 882 // Update live variables 883 LV->getVarInfo(leaInReg).Kills.push_back(NewMI); 884 LV->getVarInfo(leaOutReg).Kills.push_back(ExtMI); 885 if (isKill) 886 LV->replaceKillInstruction(Src, MI, *InsMI); 887 if (isDead) 888 LV->replaceKillInstruction(Dest, MI, *ExtMI); 889 } 890 891 return ExtMI; 892 } 893 894 /// This method must be implemented by targets that 895 /// set the M_CONVERTIBLE_TO_3_ADDR flag. When this flag is set, the target 896 /// may be able to convert a two-address instruction into a true 897 /// three-address instruction on demand. This allows the X86 target (for 898 /// example) to convert ADD and SHL instructions into LEA instructions if they 899 /// would require register copies due to two-addressness. 900 /// 901 /// This method returns a null pointer if the transformation cannot be 902 /// performed, otherwise it returns the new instruction. 903 /// 904 MachineInstr * 905 X86InstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI, 906 MachineInstr &MI, LiveVariables *LV) const { 907 // The following opcodes also sets the condition code register(s). Only 908 // convert them to equivalent lea if the condition code register def's 909 // are dead! 910 if (hasLiveCondCodeDef(MI)) 911 return nullptr; 912 913 MachineFunction &MF = *MI.getParent()->getParent(); 914 // All instructions input are two-addr instructions. Get the known operands. 915 const MachineOperand &Dest = MI.getOperand(0); 916 const MachineOperand &Src = MI.getOperand(1); 917 918 MachineInstr *NewMI = nullptr; 919 // FIXME: 16-bit LEA's are really slow on Athlons, but not bad on P4's. When 920 // we have better subtarget support, enable the 16-bit LEA generation here. 921 // 16-bit LEA is also slow on Core2. 922 bool DisableLEA16 = true; 923 bool is64Bit = Subtarget.is64Bit(); 924 925 unsigned MIOpc = MI.getOpcode(); 926 switch (MIOpc) { 927 default: return nullptr; 928 case X86::SHL64ri: { 929 assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!"); 930 unsigned ShAmt = getTruncatedShiftCount(MI, 2); 931 if (!isTruncatedShiftCountForLEA(ShAmt)) return nullptr; 932 933 // LEA can't handle RSP. 934 if (TargetRegisterInfo::isVirtualRegister(Src.getReg()) && 935 !MF.getRegInfo().constrainRegClass(Src.getReg(), 936 &X86::GR64_NOSPRegClass)) 937 return nullptr; 938 939 NewMI = BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r)) 940 .add(Dest) 941 .addReg(0) 942 .addImm(1ULL << ShAmt) 943 .add(Src) 944 .addImm(0) 945 .addReg(0); 946 break; 947 } 948 case X86::SHL32ri: { 949 assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!"); 950 unsigned ShAmt = getTruncatedShiftCount(MI, 2); 951 if (!isTruncatedShiftCountForLEA(ShAmt)) return nullptr; 952 953 unsigned Opc = is64Bit ? X86::LEA64_32r : X86::LEA32r; 954 955 // LEA can't handle ESP. 956 bool isKill, isUndef; 957 unsigned SrcReg; 958 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); 959 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/ false, 960 SrcReg, isKill, isUndef, ImplicitOp, LV)) 961 return nullptr; 962 963 MachineInstrBuilder MIB = 964 BuildMI(MF, MI.getDebugLoc(), get(Opc)) 965 .add(Dest) 966 .addReg(0) 967 .addImm(1ULL << ShAmt) 968 .addReg(SrcReg, getKillRegState(isKill) | getUndefRegState(isUndef)) 969 .addImm(0) 970 .addReg(0); 971 if (ImplicitOp.getReg() != 0) 972 MIB.add(ImplicitOp); 973 NewMI = MIB; 974 975 break; 976 } 977 case X86::SHL16ri: { 978 assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!"); 979 unsigned ShAmt = getTruncatedShiftCount(MI, 2); 980 if (!isTruncatedShiftCountForLEA(ShAmt)) return nullptr; 981 982 if (DisableLEA16) 983 return is64Bit ? convertToThreeAddressWithLEA(MIOpc, MFI, MI, LV) 984 : nullptr; 985 NewMI = BuildMI(MF, MI.getDebugLoc(), get(X86::LEA16r)) 986 .add(Dest) 987 .addReg(0) 988 .addImm(1ULL << ShAmt) 989 .add(Src) 990 .addImm(0) 991 .addReg(0); 992 break; 993 } 994 case X86::INC64r: 995 case X86::INC32r: { 996 assert(MI.getNumOperands() >= 2 && "Unknown inc instruction!"); 997 unsigned Opc = MIOpc == X86::INC64r ? X86::LEA64r 998 : (is64Bit ? X86::LEA64_32r : X86::LEA32r); 999 bool isKill, isUndef; 1000 unsigned SrcReg; 1001 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); 1002 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/ false, 1003 SrcReg, isKill, isUndef, ImplicitOp, LV)) 1004 return nullptr; 1005 1006 MachineInstrBuilder MIB = 1007 BuildMI(MF, MI.getDebugLoc(), get(Opc)) 1008 .add(Dest) 1009 .addReg(SrcReg, 1010 getKillRegState(isKill) | getUndefRegState(isUndef)); 1011 if (ImplicitOp.getReg() != 0) 1012 MIB.add(ImplicitOp); 1013 1014 NewMI = addOffset(MIB, 1); 1015 break; 1016 } 1017 case X86::INC16r: 1018 if (DisableLEA16) 1019 return is64Bit ? convertToThreeAddressWithLEA(MIOpc, MFI, MI, LV) 1020 : nullptr; 1021 assert(MI.getNumOperands() >= 2 && "Unknown inc instruction!"); 1022 NewMI = addOffset( 1023 BuildMI(MF, MI.getDebugLoc(), get(X86::LEA16r)).add(Dest).add(Src), 1); 1024 break; 1025 case X86::DEC64r: 1026 case X86::DEC32r: { 1027 assert(MI.getNumOperands() >= 2 && "Unknown dec instruction!"); 1028 unsigned Opc = MIOpc == X86::DEC64r ? X86::LEA64r 1029 : (is64Bit ? X86::LEA64_32r : X86::LEA32r); 1030 1031 bool isKill, isUndef; 1032 unsigned SrcReg; 1033 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); 1034 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/ false, 1035 SrcReg, isKill, isUndef, ImplicitOp, LV)) 1036 return nullptr; 1037 1038 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc)) 1039 .add(Dest) 1040 .addReg(SrcReg, getUndefRegState(isUndef) | 1041 getKillRegState(isKill)); 1042 if (ImplicitOp.getReg() != 0) 1043 MIB.add(ImplicitOp); 1044 1045 NewMI = addOffset(MIB, -1); 1046 1047 break; 1048 } 1049 case X86::DEC16r: 1050 if (DisableLEA16) 1051 return is64Bit ? convertToThreeAddressWithLEA(MIOpc, MFI, MI, LV) 1052 : nullptr; 1053 assert(MI.getNumOperands() >= 2 && "Unknown dec instruction!"); 1054 NewMI = addOffset( 1055 BuildMI(MF, MI.getDebugLoc(), get(X86::LEA16r)).add(Dest).add(Src), -1); 1056 break; 1057 case X86::ADD64rr: 1058 case X86::ADD64rr_DB: 1059 case X86::ADD32rr: 1060 case X86::ADD32rr_DB: { 1061 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!"); 1062 unsigned Opc; 1063 if (MIOpc == X86::ADD64rr || MIOpc == X86::ADD64rr_DB) 1064 Opc = X86::LEA64r; 1065 else 1066 Opc = is64Bit ? X86::LEA64_32r : X86::LEA32r; 1067 1068 bool isKill, isUndef; 1069 unsigned SrcReg; 1070 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); 1071 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/ true, 1072 SrcReg, isKill, isUndef, ImplicitOp, LV)) 1073 return nullptr; 1074 1075 const MachineOperand &Src2 = MI.getOperand(2); 1076 bool isKill2, isUndef2; 1077 unsigned SrcReg2; 1078 MachineOperand ImplicitOp2 = MachineOperand::CreateReg(0, false); 1079 if (!classifyLEAReg(MI, Src2, Opc, /*AllowSP=*/ false, 1080 SrcReg2, isKill2, isUndef2, ImplicitOp2, LV)) 1081 return nullptr; 1082 1083 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc)).add(Dest); 1084 if (ImplicitOp.getReg() != 0) 1085 MIB.add(ImplicitOp); 1086 if (ImplicitOp2.getReg() != 0) 1087 MIB.add(ImplicitOp2); 1088 1089 NewMI = addRegReg(MIB, SrcReg, isKill, SrcReg2, isKill2); 1090 1091 // Preserve undefness of the operands. 1092 NewMI->getOperand(1).setIsUndef(isUndef); 1093 NewMI->getOperand(3).setIsUndef(isUndef2); 1094 1095 if (LV && Src2.isKill()) 1096 LV->replaceKillInstruction(SrcReg2, MI, *NewMI); 1097 break; 1098 } 1099 case X86::ADD16rr: 1100 case X86::ADD16rr_DB: { 1101 if (DisableLEA16) 1102 return is64Bit ? convertToThreeAddressWithLEA(MIOpc, MFI, MI, LV) 1103 : nullptr; 1104 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!"); 1105 unsigned Src2 = MI.getOperand(2).getReg(); 1106 bool isKill2 = MI.getOperand(2).isKill(); 1107 NewMI = addRegReg(BuildMI(MF, MI.getDebugLoc(), get(X86::LEA16r)).add(Dest), 1108 Src.getReg(), Src.isKill(), Src2, isKill2); 1109 1110 // Preserve undefness of the operands. 1111 bool isUndef = MI.getOperand(1).isUndef(); 1112 bool isUndef2 = MI.getOperand(2).isUndef(); 1113 NewMI->getOperand(1).setIsUndef(isUndef); 1114 NewMI->getOperand(3).setIsUndef(isUndef2); 1115 1116 if (LV && isKill2) 1117 LV->replaceKillInstruction(Src2, MI, *NewMI); 1118 break; 1119 } 1120 case X86::ADD64ri32: 1121 case X86::ADD64ri8: 1122 case X86::ADD64ri32_DB: 1123 case X86::ADD64ri8_DB: 1124 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!"); 1125 NewMI = addOffset( 1126 BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r)).add(Dest).add(Src), 1127 MI.getOperand(2)); 1128 break; 1129 case X86::ADD32ri: 1130 case X86::ADD32ri8: 1131 case X86::ADD32ri_DB: 1132 case X86::ADD32ri8_DB: { 1133 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!"); 1134 unsigned Opc = is64Bit ? X86::LEA64_32r : X86::LEA32r; 1135 1136 bool isKill, isUndef; 1137 unsigned SrcReg; 1138 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false); 1139 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/ true, 1140 SrcReg, isKill, isUndef, ImplicitOp, LV)) 1141 return nullptr; 1142 1143 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc)) 1144 .add(Dest) 1145 .addReg(SrcReg, getUndefRegState(isUndef) | 1146 getKillRegState(isKill)); 1147 if (ImplicitOp.getReg() != 0) 1148 MIB.add(ImplicitOp); 1149 1150 NewMI = addOffset(MIB, MI.getOperand(2)); 1151 break; 1152 } 1153 case X86::ADD16ri: 1154 case X86::ADD16ri8: 1155 case X86::ADD16ri_DB: 1156 case X86::ADD16ri8_DB: 1157 if (DisableLEA16) 1158 return is64Bit ? convertToThreeAddressWithLEA(MIOpc, MFI, MI, LV) 1159 : nullptr; 1160 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!"); 1161 NewMI = addOffset( 1162 BuildMI(MF, MI.getDebugLoc(), get(X86::LEA16r)).add(Dest).add(Src), 1163 MI.getOperand(2)); 1164 break; 1165 1166 case X86::VMOVDQU8Z128rmk: 1167 case X86::VMOVDQU8Z256rmk: 1168 case X86::VMOVDQU8Zrmk: 1169 case X86::VMOVDQU16Z128rmk: 1170 case X86::VMOVDQU16Z256rmk: 1171 case X86::VMOVDQU16Zrmk: 1172 case X86::VMOVDQU32Z128rmk: case X86::VMOVDQA32Z128rmk: 1173 case X86::VMOVDQU32Z256rmk: case X86::VMOVDQA32Z256rmk: 1174 case X86::VMOVDQU32Zrmk: case X86::VMOVDQA32Zrmk: 1175 case X86::VMOVDQU64Z128rmk: case X86::VMOVDQA64Z128rmk: 1176 case X86::VMOVDQU64Z256rmk: case X86::VMOVDQA64Z256rmk: 1177 case X86::VMOVDQU64Zrmk: case X86::VMOVDQA64Zrmk: 1178 case X86::VMOVUPDZ128rmk: case X86::VMOVAPDZ128rmk: 1179 case X86::VMOVUPDZ256rmk: case X86::VMOVAPDZ256rmk: 1180 case X86::VMOVUPDZrmk: case X86::VMOVAPDZrmk: 1181 case X86::VMOVUPSZ128rmk: case X86::VMOVAPSZ128rmk: 1182 case X86::VMOVUPSZ256rmk: case X86::VMOVAPSZ256rmk: 1183 case X86::VMOVUPSZrmk: case X86::VMOVAPSZrmk: { 1184 unsigned Opc; 1185 switch (MIOpc) { 1186 default: llvm_unreachable("Unreachable!"); 1187 case X86::VMOVDQU8Z128rmk: Opc = X86::VPBLENDMBZ128rmk; break; 1188 case X86::VMOVDQU8Z256rmk: Opc = X86::VPBLENDMBZ256rmk; break; 1189 case X86::VMOVDQU8Zrmk: Opc = X86::VPBLENDMBZrmk; break; 1190 case X86::VMOVDQU16Z128rmk: Opc = X86::VPBLENDMWZ128rmk; break; 1191 case X86::VMOVDQU16Z256rmk: Opc = X86::VPBLENDMWZ256rmk; break; 1192 case X86::VMOVDQU16Zrmk: Opc = X86::VPBLENDMWZrmk; break; 1193 case X86::VMOVDQU32Z128rmk: Opc = X86::VPBLENDMDZ128rmk; break; 1194 case X86::VMOVDQU32Z256rmk: Opc = X86::VPBLENDMDZ256rmk; break; 1195 case X86::VMOVDQU32Zrmk: Opc = X86::VPBLENDMDZrmk; break; 1196 case X86::VMOVDQU64Z128rmk: Opc = X86::VPBLENDMQZ128rmk; break; 1197 case X86::VMOVDQU64Z256rmk: Opc = X86::VPBLENDMQZ256rmk; break; 1198 case X86::VMOVDQU64Zrmk: Opc = X86::VPBLENDMQZrmk; break; 1199 case X86::VMOVUPDZ128rmk: Opc = X86::VBLENDMPDZ128rmk; break; 1200 case X86::VMOVUPDZ256rmk: Opc = X86::VBLENDMPDZ256rmk; break; 1201 case X86::VMOVUPDZrmk: Opc = X86::VBLENDMPDZrmk; break; 1202 case X86::VMOVUPSZ128rmk: Opc = X86::VBLENDMPSZ128rmk; break; 1203 case X86::VMOVUPSZ256rmk: Opc = X86::VBLENDMPSZ256rmk; break; 1204 case X86::VMOVUPSZrmk: Opc = X86::VBLENDMPSZrmk; break; 1205 case X86::VMOVDQA32Z128rmk: Opc = X86::VPBLENDMDZ128rmk; break; 1206 case X86::VMOVDQA32Z256rmk: Opc = X86::VPBLENDMDZ256rmk; break; 1207 case X86::VMOVDQA32Zrmk: Opc = X86::VPBLENDMDZrmk; break; 1208 case X86::VMOVDQA64Z128rmk: Opc = X86::VPBLENDMQZ128rmk; break; 1209 case X86::VMOVDQA64Z256rmk: Opc = X86::VPBLENDMQZ256rmk; break; 1210 case X86::VMOVDQA64Zrmk: Opc = X86::VPBLENDMQZrmk; break; 1211 case X86::VMOVAPDZ128rmk: Opc = X86::VBLENDMPDZ128rmk; break; 1212 case X86::VMOVAPDZ256rmk: Opc = X86::VBLENDMPDZ256rmk; break; 1213 case X86::VMOVAPDZrmk: Opc = X86::VBLENDMPDZrmk; break; 1214 case X86::VMOVAPSZ128rmk: Opc = X86::VBLENDMPSZ128rmk; break; 1215 case X86::VMOVAPSZ256rmk: Opc = X86::VBLENDMPSZ256rmk; break; 1216 case X86::VMOVAPSZrmk: Opc = X86::VBLENDMPSZrmk; break; 1217 } 1218 1219 NewMI = BuildMI(MF, MI.getDebugLoc(), get(Opc)) 1220 .add(Dest) 1221 .add(MI.getOperand(2)) 1222 .add(Src) 1223 .add(MI.getOperand(3)) 1224 .add(MI.getOperand(4)) 1225 .add(MI.getOperand(5)) 1226 .add(MI.getOperand(6)) 1227 .add(MI.getOperand(7)); 1228 break; 1229 } 1230 case X86::VMOVDQU8Z128rrk: 1231 case X86::VMOVDQU8Z256rrk: 1232 case X86::VMOVDQU8Zrrk: 1233 case X86::VMOVDQU16Z128rrk: 1234 case X86::VMOVDQU16Z256rrk: 1235 case X86::VMOVDQU16Zrrk: 1236 case X86::VMOVDQU32Z128rrk: case X86::VMOVDQA32Z128rrk: 1237 case X86::VMOVDQU32Z256rrk: case X86::VMOVDQA32Z256rrk: 1238 case X86::VMOVDQU32Zrrk: case X86::VMOVDQA32Zrrk: 1239 case X86::VMOVDQU64Z128rrk: case X86::VMOVDQA64Z128rrk: 1240 case X86::VMOVDQU64Z256rrk: case X86::VMOVDQA64Z256rrk: 1241 case X86::VMOVDQU64Zrrk: case X86::VMOVDQA64Zrrk: 1242 case X86::VMOVUPDZ128rrk: case X86::VMOVAPDZ128rrk: 1243 case X86::VMOVUPDZ256rrk: case X86::VMOVAPDZ256rrk: 1244 case X86::VMOVUPDZrrk: case X86::VMOVAPDZrrk: 1245 case X86::VMOVUPSZ128rrk: case X86::VMOVAPSZ128rrk: 1246 case X86::VMOVUPSZ256rrk: case X86::VMOVAPSZ256rrk: 1247 case X86::VMOVUPSZrrk: case X86::VMOVAPSZrrk: { 1248 unsigned Opc; 1249 switch (MIOpc) { 1250 default: llvm_unreachable("Unreachable!"); 1251 case X86::VMOVDQU8Z128rrk: Opc = X86::VPBLENDMBZ128rrk; break; 1252 case X86::VMOVDQU8Z256rrk: Opc = X86::VPBLENDMBZ256rrk; break; 1253 case X86::VMOVDQU8Zrrk: Opc = X86::VPBLENDMBZrrk; break; 1254 case X86::VMOVDQU16Z128rrk: Opc = X86::VPBLENDMWZ128rrk; break; 1255 case X86::VMOVDQU16Z256rrk: Opc = X86::VPBLENDMWZ256rrk; break; 1256 case X86::VMOVDQU16Zrrk: Opc = X86::VPBLENDMWZrrk; break; 1257 case X86::VMOVDQU32Z128rrk: Opc = X86::VPBLENDMDZ128rrk; break; 1258 case X86::VMOVDQU32Z256rrk: Opc = X86::VPBLENDMDZ256rrk; break; 1259 case X86::VMOVDQU32Zrrk: Opc = X86::VPBLENDMDZrrk; break; 1260 case X86::VMOVDQU64Z128rrk: Opc = X86::VPBLENDMQZ128rrk; break; 1261 case X86::VMOVDQU64Z256rrk: Opc = X86::VPBLENDMQZ256rrk; break; 1262 case X86::VMOVDQU64Zrrk: Opc = X86::VPBLENDMQZrrk; break; 1263 case X86::VMOVUPDZ128rrk: Opc = X86::VBLENDMPDZ128rrk; break; 1264 case X86::VMOVUPDZ256rrk: Opc = X86::VBLENDMPDZ256rrk; break; 1265 case X86::VMOVUPDZrrk: Opc = X86::VBLENDMPDZrrk; break; 1266 case X86::VMOVUPSZ128rrk: Opc = X86::VBLENDMPSZ128rrk; break; 1267 case X86::VMOVUPSZ256rrk: Opc = X86::VBLENDMPSZ256rrk; break; 1268 case X86::VMOVUPSZrrk: Opc = X86::VBLENDMPSZrrk; break; 1269 case X86::VMOVDQA32Z128rrk: Opc = X86::VPBLENDMDZ128rrk; break; 1270 case X86::VMOVDQA32Z256rrk: Opc = X86::VPBLENDMDZ256rrk; break; 1271 case X86::VMOVDQA32Zrrk: Opc = X86::VPBLENDMDZrrk; break; 1272 case X86::VMOVDQA64Z128rrk: Opc = X86::VPBLENDMQZ128rrk; break; 1273 case X86::VMOVDQA64Z256rrk: Opc = X86::VPBLENDMQZ256rrk; break; 1274 case X86::VMOVDQA64Zrrk: Opc = X86::VPBLENDMQZrrk; break; 1275 case X86::VMOVAPDZ128rrk: Opc = X86::VBLENDMPDZ128rrk; break; 1276 case X86::VMOVAPDZ256rrk: Opc = X86::VBLENDMPDZ256rrk; break; 1277 case X86::VMOVAPDZrrk: Opc = X86::VBLENDMPDZrrk; break; 1278 case X86::VMOVAPSZ128rrk: Opc = X86::VBLENDMPSZ128rrk; break; 1279 case X86::VMOVAPSZ256rrk: Opc = X86::VBLENDMPSZ256rrk; break; 1280 case X86::VMOVAPSZrrk: Opc = X86::VBLENDMPSZrrk; break; 1281 } 1282 1283 NewMI = BuildMI(MF, MI.getDebugLoc(), get(Opc)) 1284 .add(Dest) 1285 .add(MI.getOperand(2)) 1286 .add(Src) 1287 .add(MI.getOperand(3)); 1288 break; 1289 } 1290 } 1291 1292 if (!NewMI) return nullptr; 1293 1294 if (LV) { // Update live variables 1295 if (Src.isKill()) 1296 LV->replaceKillInstruction(Src.getReg(), MI, *NewMI); 1297 if (Dest.isDead()) 1298 LV->replaceKillInstruction(Dest.getReg(), MI, *NewMI); 1299 } 1300 1301 MFI->insert(MI.getIterator(), NewMI); // Insert the new inst 1302 return NewMI; 1303 } 1304 1305 /// This determines which of three possible cases of a three source commute 1306 /// the source indexes correspond to taking into account any mask operands. 1307 /// All prevents commuting a passthru operand. Returns -1 if the commute isn't 1308 /// possible. 1309 /// Case 0 - Possible to commute the first and second operands. 1310 /// Case 1 - Possible to commute the first and third operands. 1311 /// Case 2 - Possible to commute the second and third operands. 1312 static unsigned getThreeSrcCommuteCase(uint64_t TSFlags, unsigned SrcOpIdx1, 1313 unsigned SrcOpIdx2) { 1314 // Put the lowest index to SrcOpIdx1 to simplify the checks below. 1315 if (SrcOpIdx1 > SrcOpIdx2) 1316 std::swap(SrcOpIdx1, SrcOpIdx2); 1317 1318 unsigned Op1 = 1, Op2 = 2, Op3 = 3; 1319 if (X86II::isKMasked(TSFlags)) { 1320 Op2++; 1321 Op3++; 1322 } 1323 1324 if (SrcOpIdx1 == Op1 && SrcOpIdx2 == Op2) 1325 return 0; 1326 if (SrcOpIdx1 == Op1 && SrcOpIdx2 == Op3) 1327 return 1; 1328 if (SrcOpIdx1 == Op2 && SrcOpIdx2 == Op3) 1329 return 2; 1330 llvm_unreachable("Unknown three src commute case."); 1331 } 1332 1333 unsigned X86InstrInfo::getFMA3OpcodeToCommuteOperands( 1334 const MachineInstr &MI, unsigned SrcOpIdx1, unsigned SrcOpIdx2, 1335 const X86InstrFMA3Group &FMA3Group) const { 1336 1337 unsigned Opc = MI.getOpcode(); 1338 1339 // TODO: Commuting the 1st operand of FMA*_Int requires some additional 1340 // analysis. The commute optimization is legal only if all users of FMA*_Int 1341 // use only the lowest element of the FMA*_Int instruction. Such analysis are 1342 // not implemented yet. So, just return 0 in that case. 1343 // When such analysis are available this place will be the right place for 1344 // calling it. 1345 assert(!(FMA3Group.isIntrinsic() && (SrcOpIdx1 == 1 || SrcOpIdx2 == 1)) && 1346 "Intrinsic instructions can't commute operand 1"); 1347 1348 // Determine which case this commute is or if it can't be done. 1349 unsigned Case = getThreeSrcCommuteCase(MI.getDesc().TSFlags, SrcOpIdx1, 1350 SrcOpIdx2); 1351 assert(Case < 3 && "Unexpected case number!"); 1352 1353 // Define the FMA forms mapping array that helps to map input FMA form 1354 // to output FMA form to preserve the operation semantics after 1355 // commuting the operands. 1356 const unsigned Form132Index = 0; 1357 const unsigned Form213Index = 1; 1358 const unsigned Form231Index = 2; 1359 static const unsigned FormMapping[][3] = { 1360 // 0: SrcOpIdx1 == 1 && SrcOpIdx2 == 2; 1361 // FMA132 A, C, b; ==> FMA231 C, A, b; 1362 // FMA213 B, A, c; ==> FMA213 A, B, c; 1363 // FMA231 C, A, b; ==> FMA132 A, C, b; 1364 { Form231Index, Form213Index, Form132Index }, 1365 // 1: SrcOpIdx1 == 1 && SrcOpIdx2 == 3; 1366 // FMA132 A, c, B; ==> FMA132 B, c, A; 1367 // FMA213 B, a, C; ==> FMA231 C, a, B; 1368 // FMA231 C, a, B; ==> FMA213 B, a, C; 1369 { Form132Index, Form231Index, Form213Index }, 1370 // 2: SrcOpIdx1 == 2 && SrcOpIdx2 == 3; 1371 // FMA132 a, C, B; ==> FMA213 a, B, C; 1372 // FMA213 b, A, C; ==> FMA132 b, C, A; 1373 // FMA231 c, A, B; ==> FMA231 c, B, A; 1374 { Form213Index, Form132Index, Form231Index } 1375 }; 1376 1377 unsigned FMAForms[3]; 1378 FMAForms[0] = FMA3Group.get132Opcode(); 1379 FMAForms[1] = FMA3Group.get213Opcode(); 1380 FMAForms[2] = FMA3Group.get231Opcode(); 1381 unsigned FormIndex; 1382 for (FormIndex = 0; FormIndex < 3; FormIndex++) 1383 if (Opc == FMAForms[FormIndex]) 1384 break; 1385 1386 // Everything is ready, just adjust the FMA opcode and return it. 1387 FormIndex = FormMapping[Case][FormIndex]; 1388 return FMAForms[FormIndex]; 1389 } 1390 1391 static void commuteVPTERNLOG(MachineInstr &MI, unsigned SrcOpIdx1, 1392 unsigned SrcOpIdx2) { 1393 // Determine which case this commute is or if it can't be done. 1394 unsigned Case = getThreeSrcCommuteCase(MI.getDesc().TSFlags, SrcOpIdx1, 1395 SrcOpIdx2); 1396 assert(Case < 3 && "Unexpected case value!"); 1397 1398 // For each case we need to swap two pairs of bits in the final immediate. 1399 static const uint8_t SwapMasks[3][4] = { 1400 { 0x04, 0x10, 0x08, 0x20 }, // Swap bits 2/4 and 3/5. 1401 { 0x02, 0x10, 0x08, 0x40 }, // Swap bits 1/4 and 3/6. 1402 { 0x02, 0x04, 0x20, 0x40 }, // Swap bits 1/2 and 5/6. 1403 }; 1404 1405 uint8_t Imm = MI.getOperand(MI.getNumOperands()-1).getImm(); 1406 // Clear out the bits we are swapping. 1407 uint8_t NewImm = Imm & ~(SwapMasks[Case][0] | SwapMasks[Case][1] | 1408 SwapMasks[Case][2] | SwapMasks[Case][3]); 1409 // If the immediate had a bit of the pair set, then set the opposite bit. 1410 if (Imm & SwapMasks[Case][0]) NewImm |= SwapMasks[Case][1]; 1411 if (Imm & SwapMasks[Case][1]) NewImm |= SwapMasks[Case][0]; 1412 if (Imm & SwapMasks[Case][2]) NewImm |= SwapMasks[Case][3]; 1413 if (Imm & SwapMasks[Case][3]) NewImm |= SwapMasks[Case][2]; 1414 MI.getOperand(MI.getNumOperands()-1).setImm(NewImm); 1415 } 1416 1417 // Returns true if this is a VPERMI2 or VPERMT2 instruction that can be 1418 // commuted. 1419 static bool isCommutableVPERMV3Instruction(unsigned Opcode) { 1420 #define VPERM_CASES(Suffix) \ 1421 case X86::VPERMI2##Suffix##128rr: case X86::VPERMT2##Suffix##128rr: \ 1422 case X86::VPERMI2##Suffix##256rr: case X86::VPERMT2##Suffix##256rr: \ 1423 case X86::VPERMI2##Suffix##rr: case X86::VPERMT2##Suffix##rr: \ 1424 case X86::VPERMI2##Suffix##128rm: case X86::VPERMT2##Suffix##128rm: \ 1425 case X86::VPERMI2##Suffix##256rm: case X86::VPERMT2##Suffix##256rm: \ 1426 case X86::VPERMI2##Suffix##rm: case X86::VPERMT2##Suffix##rm: \ 1427 case X86::VPERMI2##Suffix##128rrkz: case X86::VPERMT2##Suffix##128rrkz: \ 1428 case X86::VPERMI2##Suffix##256rrkz: case X86::VPERMT2##Suffix##256rrkz: \ 1429 case X86::VPERMI2##Suffix##rrkz: case X86::VPERMT2##Suffix##rrkz: \ 1430 case X86::VPERMI2##Suffix##128rmkz: case X86::VPERMT2##Suffix##128rmkz: \ 1431 case X86::VPERMI2##Suffix##256rmkz: case X86::VPERMT2##Suffix##256rmkz: \ 1432 case X86::VPERMI2##Suffix##rmkz: case X86::VPERMT2##Suffix##rmkz: 1433 1434 #define VPERM_CASES_BROADCAST(Suffix) \ 1435 VPERM_CASES(Suffix) \ 1436 case X86::VPERMI2##Suffix##128rmb: case X86::VPERMT2##Suffix##128rmb: \ 1437 case X86::VPERMI2##Suffix##256rmb: case X86::VPERMT2##Suffix##256rmb: \ 1438 case X86::VPERMI2##Suffix##rmb: case X86::VPERMT2##Suffix##rmb: \ 1439 case X86::VPERMI2##Suffix##128rmbkz: case X86::VPERMT2##Suffix##128rmbkz: \ 1440 case X86::VPERMI2##Suffix##256rmbkz: case X86::VPERMT2##Suffix##256rmbkz: \ 1441 case X86::VPERMI2##Suffix##rmbkz: case X86::VPERMT2##Suffix##rmbkz: 1442 1443 switch (Opcode) { 1444 default: return false; 1445 VPERM_CASES(B) 1446 VPERM_CASES_BROADCAST(D) 1447 VPERM_CASES_BROADCAST(PD) 1448 VPERM_CASES_BROADCAST(PS) 1449 VPERM_CASES_BROADCAST(Q) 1450 VPERM_CASES(W) 1451 return true; 1452 } 1453 #undef VPERM_CASES_BROADCAST 1454 #undef VPERM_CASES 1455 } 1456 1457 // Returns commuted opcode for VPERMI2 and VPERMT2 instructions by switching 1458 // from the I opcode to the T opcode and vice versa. 1459 static unsigned getCommutedVPERMV3Opcode(unsigned Opcode) { 1460 #define VPERM_CASES(Orig, New) \ 1461 case X86::Orig##128rr: return X86::New##128rr; \ 1462 case X86::Orig##128rrkz: return X86::New##128rrkz; \ 1463 case X86::Orig##128rm: return X86::New##128rm; \ 1464 case X86::Orig##128rmkz: return X86::New##128rmkz; \ 1465 case X86::Orig##256rr: return X86::New##256rr; \ 1466 case X86::Orig##256rrkz: return X86::New##256rrkz; \ 1467 case X86::Orig##256rm: return X86::New##256rm; \ 1468 case X86::Orig##256rmkz: return X86::New##256rmkz; \ 1469 case X86::Orig##rr: return X86::New##rr; \ 1470 case X86::Orig##rrkz: return X86::New##rrkz; \ 1471 case X86::Orig##rm: return X86::New##rm; \ 1472 case X86::Orig##rmkz: return X86::New##rmkz; 1473 1474 #define VPERM_CASES_BROADCAST(Orig, New) \ 1475 VPERM_CASES(Orig, New) \ 1476 case X86::Orig##128rmb: return X86::New##128rmb; \ 1477 case X86::Orig##128rmbkz: return X86::New##128rmbkz; \ 1478 case X86::Orig##256rmb: return X86::New##256rmb; \ 1479 case X86::Orig##256rmbkz: return X86::New##256rmbkz; \ 1480 case X86::Orig##rmb: return X86::New##rmb; \ 1481 case X86::Orig##rmbkz: return X86::New##rmbkz; 1482 1483 switch (Opcode) { 1484 VPERM_CASES(VPERMI2B, VPERMT2B) 1485 VPERM_CASES_BROADCAST(VPERMI2D, VPERMT2D) 1486 VPERM_CASES_BROADCAST(VPERMI2PD, VPERMT2PD) 1487 VPERM_CASES_BROADCAST(VPERMI2PS, VPERMT2PS) 1488 VPERM_CASES_BROADCAST(VPERMI2Q, VPERMT2Q) 1489 VPERM_CASES(VPERMI2W, VPERMT2W) 1490 VPERM_CASES(VPERMT2B, VPERMI2B) 1491 VPERM_CASES_BROADCAST(VPERMT2D, VPERMI2D) 1492 VPERM_CASES_BROADCAST(VPERMT2PD, VPERMI2PD) 1493 VPERM_CASES_BROADCAST(VPERMT2PS, VPERMI2PS) 1494 VPERM_CASES_BROADCAST(VPERMT2Q, VPERMI2Q) 1495 VPERM_CASES(VPERMT2W, VPERMI2W) 1496 } 1497 1498 llvm_unreachable("Unreachable!"); 1499 #undef VPERM_CASES_BROADCAST 1500 #undef VPERM_CASES 1501 } 1502 1503 MachineInstr *X86InstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI, 1504 unsigned OpIdx1, 1505 unsigned OpIdx2) const { 1506 auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & { 1507 if (NewMI) 1508 return *MI.getParent()->getParent()->CloneMachineInstr(&MI); 1509 return MI; 1510 }; 1511 1512 switch (MI.getOpcode()) { 1513 case X86::SHRD16rri8: // A = SHRD16rri8 B, C, I -> A = SHLD16rri8 C, B, (16-I) 1514 case X86::SHLD16rri8: // A = SHLD16rri8 B, C, I -> A = SHRD16rri8 C, B, (16-I) 1515 case X86::SHRD32rri8: // A = SHRD32rri8 B, C, I -> A = SHLD32rri8 C, B, (32-I) 1516 case X86::SHLD32rri8: // A = SHLD32rri8 B, C, I -> A = SHRD32rri8 C, B, (32-I) 1517 case X86::SHRD64rri8: // A = SHRD64rri8 B, C, I -> A = SHLD64rri8 C, B, (64-I) 1518 case X86::SHLD64rri8:{// A = SHLD64rri8 B, C, I -> A = SHRD64rri8 C, B, (64-I) 1519 unsigned Opc; 1520 unsigned Size; 1521 switch (MI.getOpcode()) { 1522 default: llvm_unreachable("Unreachable!"); 1523 case X86::SHRD16rri8: Size = 16; Opc = X86::SHLD16rri8; break; 1524 case X86::SHLD16rri8: Size = 16; Opc = X86::SHRD16rri8; break; 1525 case X86::SHRD32rri8: Size = 32; Opc = X86::SHLD32rri8; break; 1526 case X86::SHLD32rri8: Size = 32; Opc = X86::SHRD32rri8; break; 1527 case X86::SHRD64rri8: Size = 64; Opc = X86::SHLD64rri8; break; 1528 case X86::SHLD64rri8: Size = 64; Opc = X86::SHRD64rri8; break; 1529 } 1530 unsigned Amt = MI.getOperand(3).getImm(); 1531 auto &WorkingMI = cloneIfNew(MI); 1532 WorkingMI.setDesc(get(Opc)); 1533 WorkingMI.getOperand(3).setImm(Size - Amt); 1534 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 1535 OpIdx1, OpIdx2); 1536 } 1537 case X86::PFSUBrr: 1538 case X86::PFSUBRrr: { 1539 // PFSUB x, y: x = x - y 1540 // PFSUBR x, y: x = y - x 1541 unsigned Opc = 1542 (X86::PFSUBRrr == MI.getOpcode() ? X86::PFSUBrr : X86::PFSUBRrr); 1543 auto &WorkingMI = cloneIfNew(MI); 1544 WorkingMI.setDesc(get(Opc)); 1545 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 1546 OpIdx1, OpIdx2); 1547 } 1548 case X86::BLENDPDrri: 1549 case X86::BLENDPSrri: 1550 case X86::VBLENDPDrri: 1551 case X86::VBLENDPSrri: 1552 // If we're optimizing for size, try to use MOVSD/MOVSS. 1553 if (MI.getParent()->getParent()->getFunction().optForSize()) { 1554 unsigned Mask, Opc; 1555 switch (MI.getOpcode()) { 1556 default: llvm_unreachable("Unreachable!"); 1557 case X86::BLENDPDrri: Opc = X86::MOVSDrr; Mask = 0x03; break; 1558 case X86::BLENDPSrri: Opc = X86::MOVSSrr; Mask = 0x0F; break; 1559 case X86::VBLENDPDrri: Opc = X86::VMOVSDrr; Mask = 0x03; break; 1560 case X86::VBLENDPSrri: Opc = X86::VMOVSSrr; Mask = 0x0F; break; 1561 } 1562 if ((MI.getOperand(3).getImm() ^ Mask) == 1) { 1563 auto &WorkingMI = cloneIfNew(MI); 1564 WorkingMI.setDesc(get(Opc)); 1565 WorkingMI.RemoveOperand(3); 1566 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, 1567 /*NewMI=*/false, 1568 OpIdx1, OpIdx2); 1569 } 1570 } 1571 LLVM_FALLTHROUGH; 1572 case X86::PBLENDWrri: 1573 case X86::VBLENDPDYrri: 1574 case X86::VBLENDPSYrri: 1575 case X86::VPBLENDDrri: 1576 case X86::VPBLENDWrri: 1577 case X86::VPBLENDDYrri: 1578 case X86::VPBLENDWYrri:{ 1579 unsigned Mask; 1580 switch (MI.getOpcode()) { 1581 default: llvm_unreachable("Unreachable!"); 1582 case X86::BLENDPDrri: Mask = 0x03; break; 1583 case X86::BLENDPSrri: Mask = 0x0F; break; 1584 case X86::PBLENDWrri: Mask = 0xFF; break; 1585 case X86::VBLENDPDrri: Mask = 0x03; break; 1586 case X86::VBLENDPSrri: Mask = 0x0F; break; 1587 case X86::VBLENDPDYrri: Mask = 0x0F; break; 1588 case X86::VBLENDPSYrri: Mask = 0xFF; break; 1589 case X86::VPBLENDDrri: Mask = 0x0F; break; 1590 case X86::VPBLENDWrri: Mask = 0xFF; break; 1591 case X86::VPBLENDDYrri: Mask = 0xFF; break; 1592 case X86::VPBLENDWYrri: Mask = 0xFF; break; 1593 } 1594 // Only the least significant bits of Imm are used. 1595 unsigned Imm = MI.getOperand(3).getImm() & Mask; 1596 auto &WorkingMI = cloneIfNew(MI); 1597 WorkingMI.getOperand(3).setImm(Mask ^ Imm); 1598 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 1599 OpIdx1, OpIdx2); 1600 } 1601 case X86::MOVSDrr: 1602 case X86::MOVSSrr: 1603 case X86::VMOVSDrr: 1604 case X86::VMOVSSrr:{ 1605 // On SSE41 or later we can commute a MOVSS/MOVSD to a BLENDPS/BLENDPD. 1606 assert(Subtarget.hasSSE41() && "Commuting MOVSD/MOVSS requires SSE41!"); 1607 1608 unsigned Mask, Opc; 1609 switch (MI.getOpcode()) { 1610 default: llvm_unreachable("Unreachable!"); 1611 case X86::MOVSDrr: Opc = X86::BLENDPDrri; Mask = 0x02; break; 1612 case X86::MOVSSrr: Opc = X86::BLENDPSrri; Mask = 0x0E; break; 1613 case X86::VMOVSDrr: Opc = X86::VBLENDPDrri; Mask = 0x02; break; 1614 case X86::VMOVSSrr: Opc = X86::VBLENDPSrri; Mask = 0x0E; break; 1615 } 1616 1617 auto &WorkingMI = cloneIfNew(MI); 1618 WorkingMI.setDesc(get(Opc)); 1619 WorkingMI.addOperand(MachineOperand::CreateImm(Mask)); 1620 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 1621 OpIdx1, OpIdx2); 1622 } 1623 case X86::PCLMULQDQrr: 1624 case X86::VPCLMULQDQrr: 1625 case X86::VPCLMULQDQYrr: 1626 case X86::VPCLMULQDQZrr: 1627 case X86::VPCLMULQDQZ128rr: 1628 case X86::VPCLMULQDQZ256rr: { 1629 // SRC1 64bits = Imm[0] ? SRC1[127:64] : SRC1[63:0] 1630 // SRC2 64bits = Imm[4] ? SRC2[127:64] : SRC2[63:0] 1631 unsigned Imm = MI.getOperand(3).getImm(); 1632 unsigned Src1Hi = Imm & 0x01; 1633 unsigned Src2Hi = Imm & 0x10; 1634 auto &WorkingMI = cloneIfNew(MI); 1635 WorkingMI.getOperand(3).setImm((Src1Hi << 4) | (Src2Hi >> 4)); 1636 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 1637 OpIdx1, OpIdx2); 1638 } 1639 case X86::VPCMPBZ128rri: case X86::VPCMPUBZ128rri: 1640 case X86::VPCMPBZ256rri: case X86::VPCMPUBZ256rri: 1641 case X86::VPCMPBZrri: case X86::VPCMPUBZrri: 1642 case X86::VPCMPDZ128rri: case X86::VPCMPUDZ128rri: 1643 case X86::VPCMPDZ256rri: case X86::VPCMPUDZ256rri: 1644 case X86::VPCMPDZrri: case X86::VPCMPUDZrri: 1645 case X86::VPCMPQZ128rri: case X86::VPCMPUQZ128rri: 1646 case X86::VPCMPQZ256rri: case X86::VPCMPUQZ256rri: 1647 case X86::VPCMPQZrri: case X86::VPCMPUQZrri: 1648 case X86::VPCMPWZ128rri: case X86::VPCMPUWZ128rri: 1649 case X86::VPCMPWZ256rri: case X86::VPCMPUWZ256rri: 1650 case X86::VPCMPWZrri: case X86::VPCMPUWZrri: 1651 case X86::VPCMPBZ128rrik: case X86::VPCMPUBZ128rrik: 1652 case X86::VPCMPBZ256rrik: case X86::VPCMPUBZ256rrik: 1653 case X86::VPCMPBZrrik: case X86::VPCMPUBZrrik: 1654 case X86::VPCMPDZ128rrik: case X86::VPCMPUDZ128rrik: 1655 case X86::VPCMPDZ256rrik: case X86::VPCMPUDZ256rrik: 1656 case X86::VPCMPDZrrik: case X86::VPCMPUDZrrik: 1657 case X86::VPCMPQZ128rrik: case X86::VPCMPUQZ128rrik: 1658 case X86::VPCMPQZ256rrik: case X86::VPCMPUQZ256rrik: 1659 case X86::VPCMPQZrrik: case X86::VPCMPUQZrrik: 1660 case X86::VPCMPWZ128rrik: case X86::VPCMPUWZ128rrik: 1661 case X86::VPCMPWZ256rrik: case X86::VPCMPUWZ256rrik: 1662 case X86::VPCMPWZrrik: case X86::VPCMPUWZrrik: { 1663 // Flip comparison mode immediate (if necessary). 1664 unsigned Imm = MI.getOperand(MI.getNumOperands() - 1).getImm() & 0x7; 1665 Imm = X86::getSwappedVPCMPImm(Imm); 1666 auto &WorkingMI = cloneIfNew(MI); 1667 WorkingMI.getOperand(MI.getNumOperands() - 1).setImm(Imm); 1668 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 1669 OpIdx1, OpIdx2); 1670 } 1671 case X86::VPCOMBri: case X86::VPCOMUBri: 1672 case X86::VPCOMDri: case X86::VPCOMUDri: 1673 case X86::VPCOMQri: case X86::VPCOMUQri: 1674 case X86::VPCOMWri: case X86::VPCOMUWri: { 1675 // Flip comparison mode immediate (if necessary). 1676 unsigned Imm = MI.getOperand(3).getImm() & 0x7; 1677 Imm = X86::getSwappedVPCOMImm(Imm); 1678 auto &WorkingMI = cloneIfNew(MI); 1679 WorkingMI.getOperand(3).setImm(Imm); 1680 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 1681 OpIdx1, OpIdx2); 1682 } 1683 case X86::VPERM2F128rr: 1684 case X86::VPERM2I128rr: { 1685 // Flip permute source immediate. 1686 // Imm & 0x02: lo = if set, select Op1.lo/hi else Op0.lo/hi. 1687 // Imm & 0x20: hi = if set, select Op1.lo/hi else Op0.lo/hi. 1688 unsigned Imm = MI.getOperand(3).getImm() & 0xFF; 1689 auto &WorkingMI = cloneIfNew(MI); 1690 WorkingMI.getOperand(3).setImm(Imm ^ 0x22); 1691 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 1692 OpIdx1, OpIdx2); 1693 } 1694 case X86::MOVHLPSrr: 1695 case X86::UNPCKHPDrr: 1696 case X86::VMOVHLPSrr: 1697 case X86::VUNPCKHPDrr: 1698 case X86::VMOVHLPSZrr: 1699 case X86::VUNPCKHPDZ128rr: { 1700 assert(Subtarget.hasSSE2() && "Commuting MOVHLP/UNPCKHPD requires SSE2!"); 1701 1702 unsigned Opc = MI.getOpcode(); 1703 switch (Opc) { 1704 default: llvm_unreachable("Unreachable!"); 1705 case X86::MOVHLPSrr: Opc = X86::UNPCKHPDrr; break; 1706 case X86::UNPCKHPDrr: Opc = X86::MOVHLPSrr; break; 1707 case X86::VMOVHLPSrr: Opc = X86::VUNPCKHPDrr; break; 1708 case X86::VUNPCKHPDrr: Opc = X86::VMOVHLPSrr; break; 1709 case X86::VMOVHLPSZrr: Opc = X86::VUNPCKHPDZ128rr; break; 1710 case X86::VUNPCKHPDZ128rr: Opc = X86::VMOVHLPSZrr; break; 1711 } 1712 auto &WorkingMI = cloneIfNew(MI); 1713 WorkingMI.setDesc(get(Opc)); 1714 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 1715 OpIdx1, OpIdx2); 1716 } 1717 case X86::CMOVB16rr: case X86::CMOVB32rr: case X86::CMOVB64rr: 1718 case X86::CMOVAE16rr: case X86::CMOVAE32rr: case X86::CMOVAE64rr: 1719 case X86::CMOVE16rr: case X86::CMOVE32rr: case X86::CMOVE64rr: 1720 case X86::CMOVNE16rr: case X86::CMOVNE32rr: case X86::CMOVNE64rr: 1721 case X86::CMOVBE16rr: case X86::CMOVBE32rr: case X86::CMOVBE64rr: 1722 case X86::CMOVA16rr: case X86::CMOVA32rr: case X86::CMOVA64rr: 1723 case X86::CMOVL16rr: case X86::CMOVL32rr: case X86::CMOVL64rr: 1724 case X86::CMOVGE16rr: case X86::CMOVGE32rr: case X86::CMOVGE64rr: 1725 case X86::CMOVLE16rr: case X86::CMOVLE32rr: case X86::CMOVLE64rr: 1726 case X86::CMOVG16rr: case X86::CMOVG32rr: case X86::CMOVG64rr: 1727 case X86::CMOVS16rr: case X86::CMOVS32rr: case X86::CMOVS64rr: 1728 case X86::CMOVNS16rr: case X86::CMOVNS32rr: case X86::CMOVNS64rr: 1729 case X86::CMOVP16rr: case X86::CMOVP32rr: case X86::CMOVP64rr: 1730 case X86::CMOVNP16rr: case X86::CMOVNP32rr: case X86::CMOVNP64rr: 1731 case X86::CMOVO16rr: case X86::CMOVO32rr: case X86::CMOVO64rr: 1732 case X86::CMOVNO16rr: case X86::CMOVNO32rr: case X86::CMOVNO64rr: { 1733 unsigned Opc; 1734 switch (MI.getOpcode()) { 1735 default: llvm_unreachable("Unreachable!"); 1736 case X86::CMOVB16rr: Opc = X86::CMOVAE16rr; break; 1737 case X86::CMOVB32rr: Opc = X86::CMOVAE32rr; break; 1738 case X86::CMOVB64rr: Opc = X86::CMOVAE64rr; break; 1739 case X86::CMOVAE16rr: Opc = X86::CMOVB16rr; break; 1740 case X86::CMOVAE32rr: Opc = X86::CMOVB32rr; break; 1741 case X86::CMOVAE64rr: Opc = X86::CMOVB64rr; break; 1742 case X86::CMOVE16rr: Opc = X86::CMOVNE16rr; break; 1743 case X86::CMOVE32rr: Opc = X86::CMOVNE32rr; break; 1744 case X86::CMOVE64rr: Opc = X86::CMOVNE64rr; break; 1745 case X86::CMOVNE16rr: Opc = X86::CMOVE16rr; break; 1746 case X86::CMOVNE32rr: Opc = X86::CMOVE32rr; break; 1747 case X86::CMOVNE64rr: Opc = X86::CMOVE64rr; break; 1748 case X86::CMOVBE16rr: Opc = X86::CMOVA16rr; break; 1749 case X86::CMOVBE32rr: Opc = X86::CMOVA32rr; break; 1750 case X86::CMOVBE64rr: Opc = X86::CMOVA64rr; break; 1751 case X86::CMOVA16rr: Opc = X86::CMOVBE16rr; break; 1752 case X86::CMOVA32rr: Opc = X86::CMOVBE32rr; break; 1753 case X86::CMOVA64rr: Opc = X86::CMOVBE64rr; break; 1754 case X86::CMOVL16rr: Opc = X86::CMOVGE16rr; break; 1755 case X86::CMOVL32rr: Opc = X86::CMOVGE32rr; break; 1756 case X86::CMOVL64rr: Opc = X86::CMOVGE64rr; break; 1757 case X86::CMOVGE16rr: Opc = X86::CMOVL16rr; break; 1758 case X86::CMOVGE32rr: Opc = X86::CMOVL32rr; break; 1759 case X86::CMOVGE64rr: Opc = X86::CMOVL64rr; break; 1760 case X86::CMOVLE16rr: Opc = X86::CMOVG16rr; break; 1761 case X86::CMOVLE32rr: Opc = X86::CMOVG32rr; break; 1762 case X86::CMOVLE64rr: Opc = X86::CMOVG64rr; break; 1763 case X86::CMOVG16rr: Opc = X86::CMOVLE16rr; break; 1764 case X86::CMOVG32rr: Opc = X86::CMOVLE32rr; break; 1765 case X86::CMOVG64rr: Opc = X86::CMOVLE64rr; break; 1766 case X86::CMOVS16rr: Opc = X86::CMOVNS16rr; break; 1767 case X86::CMOVS32rr: Opc = X86::CMOVNS32rr; break; 1768 case X86::CMOVS64rr: Opc = X86::CMOVNS64rr; break; 1769 case X86::CMOVNS16rr: Opc = X86::CMOVS16rr; break; 1770 case X86::CMOVNS32rr: Opc = X86::CMOVS32rr; break; 1771 case X86::CMOVNS64rr: Opc = X86::CMOVS64rr; break; 1772 case X86::CMOVP16rr: Opc = X86::CMOVNP16rr; break; 1773 case X86::CMOVP32rr: Opc = X86::CMOVNP32rr; break; 1774 case X86::CMOVP64rr: Opc = X86::CMOVNP64rr; break; 1775 case X86::CMOVNP16rr: Opc = X86::CMOVP16rr; break; 1776 case X86::CMOVNP32rr: Opc = X86::CMOVP32rr; break; 1777 case X86::CMOVNP64rr: Opc = X86::CMOVP64rr; break; 1778 case X86::CMOVO16rr: Opc = X86::CMOVNO16rr; break; 1779 case X86::CMOVO32rr: Opc = X86::CMOVNO32rr; break; 1780 case X86::CMOVO64rr: Opc = X86::CMOVNO64rr; break; 1781 case X86::CMOVNO16rr: Opc = X86::CMOVO16rr; break; 1782 case X86::CMOVNO32rr: Opc = X86::CMOVO32rr; break; 1783 case X86::CMOVNO64rr: Opc = X86::CMOVO64rr; break; 1784 } 1785 auto &WorkingMI = cloneIfNew(MI); 1786 WorkingMI.setDesc(get(Opc)); 1787 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 1788 OpIdx1, OpIdx2); 1789 } 1790 case X86::VPTERNLOGDZrri: case X86::VPTERNLOGDZrmi: 1791 case X86::VPTERNLOGDZ128rri: case X86::VPTERNLOGDZ128rmi: 1792 case X86::VPTERNLOGDZ256rri: case X86::VPTERNLOGDZ256rmi: 1793 case X86::VPTERNLOGQZrri: case X86::VPTERNLOGQZrmi: 1794 case X86::VPTERNLOGQZ128rri: case X86::VPTERNLOGQZ128rmi: 1795 case X86::VPTERNLOGQZ256rri: case X86::VPTERNLOGQZ256rmi: 1796 case X86::VPTERNLOGDZrrik: 1797 case X86::VPTERNLOGDZ128rrik: 1798 case X86::VPTERNLOGDZ256rrik: 1799 case X86::VPTERNLOGQZrrik: 1800 case X86::VPTERNLOGQZ128rrik: 1801 case X86::VPTERNLOGQZ256rrik: 1802 case X86::VPTERNLOGDZrrikz: case X86::VPTERNLOGDZrmikz: 1803 case X86::VPTERNLOGDZ128rrikz: case X86::VPTERNLOGDZ128rmikz: 1804 case X86::VPTERNLOGDZ256rrikz: case X86::VPTERNLOGDZ256rmikz: 1805 case X86::VPTERNLOGQZrrikz: case X86::VPTERNLOGQZrmikz: 1806 case X86::VPTERNLOGQZ128rrikz: case X86::VPTERNLOGQZ128rmikz: 1807 case X86::VPTERNLOGQZ256rrikz: case X86::VPTERNLOGQZ256rmikz: 1808 case X86::VPTERNLOGDZ128rmbi: 1809 case X86::VPTERNLOGDZ256rmbi: 1810 case X86::VPTERNLOGDZrmbi: 1811 case X86::VPTERNLOGQZ128rmbi: 1812 case X86::VPTERNLOGQZ256rmbi: 1813 case X86::VPTERNLOGQZrmbi: 1814 case X86::VPTERNLOGDZ128rmbikz: 1815 case X86::VPTERNLOGDZ256rmbikz: 1816 case X86::VPTERNLOGDZrmbikz: 1817 case X86::VPTERNLOGQZ128rmbikz: 1818 case X86::VPTERNLOGQZ256rmbikz: 1819 case X86::VPTERNLOGQZrmbikz: { 1820 auto &WorkingMI = cloneIfNew(MI); 1821 commuteVPTERNLOG(WorkingMI, OpIdx1, OpIdx2); 1822 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 1823 OpIdx1, OpIdx2); 1824 } 1825 default: { 1826 if (isCommutableVPERMV3Instruction(MI.getOpcode())) { 1827 unsigned Opc = getCommutedVPERMV3Opcode(MI.getOpcode()); 1828 auto &WorkingMI = cloneIfNew(MI); 1829 WorkingMI.setDesc(get(Opc)); 1830 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 1831 OpIdx1, OpIdx2); 1832 } 1833 1834 const X86InstrFMA3Group *FMA3Group = getFMA3Group(MI.getOpcode(), 1835 MI.getDesc().TSFlags); 1836 if (FMA3Group) { 1837 unsigned Opc = 1838 getFMA3OpcodeToCommuteOperands(MI, OpIdx1, OpIdx2, *FMA3Group); 1839 auto &WorkingMI = cloneIfNew(MI); 1840 WorkingMI.setDesc(get(Opc)); 1841 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false, 1842 OpIdx1, OpIdx2); 1843 } 1844 1845 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2); 1846 } 1847 } 1848 } 1849 1850 bool 1851 X86InstrInfo::findThreeSrcCommutedOpIndices(const MachineInstr &MI, 1852 unsigned &SrcOpIdx1, 1853 unsigned &SrcOpIdx2, 1854 bool IsIntrinsic) const { 1855 uint64_t TSFlags = MI.getDesc().TSFlags; 1856 1857 unsigned FirstCommutableVecOp = 1; 1858 unsigned LastCommutableVecOp = 3; 1859 unsigned KMaskOp = -1U; 1860 if (X86II::isKMasked(TSFlags)) { 1861 // For k-zero-masked operations it is Ok to commute the first vector 1862 // operand. 1863 // For regular k-masked operations a conservative choice is done as the 1864 // elements of the first vector operand, for which the corresponding bit 1865 // in the k-mask operand is set to 0, are copied to the result of the 1866 // instruction. 1867 // TODO/FIXME: The commute still may be legal if it is known that the 1868 // k-mask operand is set to either all ones or all zeroes. 1869 // It is also Ok to commute the 1st operand if all users of MI use only 1870 // the elements enabled by the k-mask operand. For example, 1871 // v4 = VFMADD213PSZrk v1, k, v2, v3; // v1[i] = k[i] ? v2[i]*v1[i]+v3[i] 1872 // : v1[i]; 1873 // VMOVAPSZmrk <mem_addr>, k, v4; // this is the ONLY user of v4 -> 1874 // // Ok, to commute v1 in FMADD213PSZrk. 1875 1876 // The k-mask operand has index = 2 for masked and zero-masked operations. 1877 KMaskOp = 2; 1878 1879 // The operand with index = 1 is used as a source for those elements for 1880 // which the corresponding bit in the k-mask is set to 0. 1881 if (X86II::isKMergeMasked(TSFlags)) 1882 FirstCommutableVecOp = 3; 1883 1884 LastCommutableVecOp++; 1885 } else if (IsIntrinsic) { 1886 // Commuting the first operand of an intrinsic instruction isn't possible 1887 // unless we can prove that only the lowest element of the result is used. 1888 FirstCommutableVecOp = 2; 1889 } 1890 1891 if (isMem(MI, LastCommutableVecOp)) 1892 LastCommutableVecOp--; 1893 1894 // Only the first RegOpsNum operands are commutable. 1895 // Also, the value 'CommuteAnyOperandIndex' is valid here as it means 1896 // that the operand is not specified/fixed. 1897 if (SrcOpIdx1 != CommuteAnyOperandIndex && 1898 (SrcOpIdx1 < FirstCommutableVecOp || SrcOpIdx1 > LastCommutableVecOp || 1899 SrcOpIdx1 == KMaskOp)) 1900 return false; 1901 if (SrcOpIdx2 != CommuteAnyOperandIndex && 1902 (SrcOpIdx2 < FirstCommutableVecOp || SrcOpIdx2 > LastCommutableVecOp || 1903 SrcOpIdx2 == KMaskOp)) 1904 return false; 1905 1906 // Look for two different register operands assumed to be commutable 1907 // regardless of the FMA opcode. The FMA opcode is adjusted later. 1908 if (SrcOpIdx1 == CommuteAnyOperandIndex || 1909 SrcOpIdx2 == CommuteAnyOperandIndex) { 1910 unsigned CommutableOpIdx1 = SrcOpIdx1; 1911 unsigned CommutableOpIdx2 = SrcOpIdx2; 1912 1913 // At least one of operands to be commuted is not specified and 1914 // this method is free to choose appropriate commutable operands. 1915 if (SrcOpIdx1 == SrcOpIdx2) 1916 // Both of operands are not fixed. By default set one of commutable 1917 // operands to the last register operand of the instruction. 1918 CommutableOpIdx2 = LastCommutableVecOp; 1919 else if (SrcOpIdx2 == CommuteAnyOperandIndex) 1920 // Only one of operands is not fixed. 1921 CommutableOpIdx2 = SrcOpIdx1; 1922 1923 // CommutableOpIdx2 is well defined now. Let's choose another commutable 1924 // operand and assign its index to CommutableOpIdx1. 1925 unsigned Op2Reg = MI.getOperand(CommutableOpIdx2).getReg(); 1926 for (CommutableOpIdx1 = LastCommutableVecOp; 1927 CommutableOpIdx1 >= FirstCommutableVecOp; CommutableOpIdx1--) { 1928 // Just ignore and skip the k-mask operand. 1929 if (CommutableOpIdx1 == KMaskOp) 1930 continue; 1931 1932 // The commuted operands must have different registers. 1933 // Otherwise, the commute transformation does not change anything and 1934 // is useless then. 1935 if (Op2Reg != MI.getOperand(CommutableOpIdx1).getReg()) 1936 break; 1937 } 1938 1939 // No appropriate commutable operands were found. 1940 if (CommutableOpIdx1 < FirstCommutableVecOp) 1941 return false; 1942 1943 // Assign the found pair of commutable indices to SrcOpIdx1 and SrcOpidx2 1944 // to return those values. 1945 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1946 CommutableOpIdx1, CommutableOpIdx2)) 1947 return false; 1948 } 1949 1950 return true; 1951 } 1952 1953 bool X86InstrInfo::findCommutedOpIndices(MachineInstr &MI, unsigned &SrcOpIdx1, 1954 unsigned &SrcOpIdx2) const { 1955 const MCInstrDesc &Desc = MI.getDesc(); 1956 if (!Desc.isCommutable()) 1957 return false; 1958 1959 switch (MI.getOpcode()) { 1960 case X86::CMPSDrr: 1961 case X86::CMPSSrr: 1962 case X86::CMPPDrri: 1963 case X86::CMPPSrri: 1964 case X86::VCMPSDrr: 1965 case X86::VCMPSSrr: 1966 case X86::VCMPPDrri: 1967 case X86::VCMPPSrri: 1968 case X86::VCMPPDYrri: 1969 case X86::VCMPPSYrri: 1970 case X86::VCMPSDZrr: 1971 case X86::VCMPSSZrr: 1972 case X86::VCMPPDZrri: 1973 case X86::VCMPPSZrri: 1974 case X86::VCMPPDZ128rri: 1975 case X86::VCMPPSZ128rri: 1976 case X86::VCMPPDZ256rri: 1977 case X86::VCMPPSZ256rri: { 1978 // Float comparison can be safely commuted for 1979 // Ordered/Unordered/Equal/NotEqual tests 1980 unsigned Imm = MI.getOperand(3).getImm() & 0x7; 1981 switch (Imm) { 1982 case 0x00: // EQUAL 1983 case 0x03: // UNORDERED 1984 case 0x04: // NOT EQUAL 1985 case 0x07: // ORDERED 1986 // The indices of the commutable operands are 1 and 2. 1987 // Assign them to the returned operand indices here. 1988 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1, 2); 1989 } 1990 return false; 1991 } 1992 case X86::MOVSDrr: 1993 case X86::MOVSSrr: 1994 case X86::VMOVSDrr: 1995 case X86::VMOVSSrr: 1996 if (Subtarget.hasSSE41()) 1997 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); 1998 return false; 1999 case X86::MOVHLPSrr: 2000 case X86::UNPCKHPDrr: 2001 case X86::VMOVHLPSrr: 2002 case X86::VUNPCKHPDrr: 2003 case X86::VMOVHLPSZrr: 2004 case X86::VUNPCKHPDZ128rr: 2005 if (Subtarget.hasSSE2()) 2006 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); 2007 return false; 2008 case X86::VPTERNLOGDZrri: case X86::VPTERNLOGDZrmi: 2009 case X86::VPTERNLOGDZ128rri: case X86::VPTERNLOGDZ128rmi: 2010 case X86::VPTERNLOGDZ256rri: case X86::VPTERNLOGDZ256rmi: 2011 case X86::VPTERNLOGQZrri: case X86::VPTERNLOGQZrmi: 2012 case X86::VPTERNLOGQZ128rri: case X86::VPTERNLOGQZ128rmi: 2013 case X86::VPTERNLOGQZ256rri: case X86::VPTERNLOGQZ256rmi: 2014 case X86::VPTERNLOGDZrrik: 2015 case X86::VPTERNLOGDZ128rrik: 2016 case X86::VPTERNLOGDZ256rrik: 2017 case X86::VPTERNLOGQZrrik: 2018 case X86::VPTERNLOGQZ128rrik: 2019 case X86::VPTERNLOGQZ256rrik: 2020 case X86::VPTERNLOGDZrrikz: case X86::VPTERNLOGDZrmikz: 2021 case X86::VPTERNLOGDZ128rrikz: case X86::VPTERNLOGDZ128rmikz: 2022 case X86::VPTERNLOGDZ256rrikz: case X86::VPTERNLOGDZ256rmikz: 2023 case X86::VPTERNLOGQZrrikz: case X86::VPTERNLOGQZrmikz: 2024 case X86::VPTERNLOGQZ128rrikz: case X86::VPTERNLOGQZ128rmikz: 2025 case X86::VPTERNLOGQZ256rrikz: case X86::VPTERNLOGQZ256rmikz: 2026 case X86::VPTERNLOGDZ128rmbi: 2027 case X86::VPTERNLOGDZ256rmbi: 2028 case X86::VPTERNLOGDZrmbi: 2029 case X86::VPTERNLOGQZ128rmbi: 2030 case X86::VPTERNLOGQZ256rmbi: 2031 case X86::VPTERNLOGQZrmbi: 2032 case X86::VPTERNLOGDZ128rmbikz: 2033 case X86::VPTERNLOGDZ256rmbikz: 2034 case X86::VPTERNLOGDZrmbikz: 2035 case X86::VPTERNLOGQZ128rmbikz: 2036 case X86::VPTERNLOGQZ256rmbikz: 2037 case X86::VPTERNLOGQZrmbikz: 2038 return findThreeSrcCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); 2039 case X86::VPMADD52HUQZ128r: 2040 case X86::VPMADD52HUQZ128rk: 2041 case X86::VPMADD52HUQZ128rkz: 2042 case X86::VPMADD52HUQZ256r: 2043 case X86::VPMADD52HUQZ256rk: 2044 case X86::VPMADD52HUQZ256rkz: 2045 case X86::VPMADD52HUQZr: 2046 case X86::VPMADD52HUQZrk: 2047 case X86::VPMADD52HUQZrkz: 2048 case X86::VPMADD52LUQZ128r: 2049 case X86::VPMADD52LUQZ128rk: 2050 case X86::VPMADD52LUQZ128rkz: 2051 case X86::VPMADD52LUQZ256r: 2052 case X86::VPMADD52LUQZ256rk: 2053 case X86::VPMADD52LUQZ256rkz: 2054 case X86::VPMADD52LUQZr: 2055 case X86::VPMADD52LUQZrk: 2056 case X86::VPMADD52LUQZrkz: { 2057 unsigned CommutableOpIdx1 = 2; 2058 unsigned CommutableOpIdx2 = 3; 2059 if (X86II::isKMasked(Desc.TSFlags)) { 2060 // Skip the mask register. 2061 ++CommutableOpIdx1; 2062 ++CommutableOpIdx2; 2063 } 2064 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2065 CommutableOpIdx1, CommutableOpIdx2)) 2066 return false; 2067 if (!MI.getOperand(SrcOpIdx1).isReg() || 2068 !MI.getOperand(SrcOpIdx2).isReg()) 2069 // No idea. 2070 return false; 2071 return true; 2072 } 2073 2074 default: 2075 const X86InstrFMA3Group *FMA3Group = getFMA3Group(MI.getOpcode(), 2076 MI.getDesc().TSFlags); 2077 if (FMA3Group) 2078 return findThreeSrcCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2, 2079 FMA3Group->isIntrinsic()); 2080 2081 // Handled masked instructions since we need to skip over the mask input 2082 // and the preserved input. 2083 if (X86II::isKMasked(Desc.TSFlags)) { 2084 // First assume that the first input is the mask operand and skip past it. 2085 unsigned CommutableOpIdx1 = Desc.getNumDefs() + 1; 2086 unsigned CommutableOpIdx2 = Desc.getNumDefs() + 2; 2087 // Check if the first input is tied. If there isn't one then we only 2088 // need to skip the mask operand which we did above. 2089 if ((MI.getDesc().getOperandConstraint(Desc.getNumDefs(), 2090 MCOI::TIED_TO) != -1)) { 2091 // If this is zero masking instruction with a tied operand, we need to 2092 // move the first index back to the first input since this must 2093 // be a 3 input instruction and we want the first two non-mask inputs. 2094 // Otherwise this is a 2 input instruction with a preserved input and 2095 // mask, so we need to move the indices to skip one more input. 2096 if (X86II::isKMergeMasked(Desc.TSFlags)) { 2097 ++CommutableOpIdx1; 2098 ++CommutableOpIdx2; 2099 } else { 2100 --CommutableOpIdx1; 2101 } 2102 } 2103 2104 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2105 CommutableOpIdx1, CommutableOpIdx2)) 2106 return false; 2107 2108 if (!MI.getOperand(SrcOpIdx1).isReg() || 2109 !MI.getOperand(SrcOpIdx2).isReg()) 2110 // No idea. 2111 return false; 2112 return true; 2113 } 2114 2115 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); 2116 } 2117 return false; 2118 } 2119 2120 X86::CondCode X86::getCondFromBranchOpc(unsigned BrOpc) { 2121 switch (BrOpc) { 2122 default: return X86::COND_INVALID; 2123 case X86::JE_1: return X86::COND_E; 2124 case X86::JNE_1: return X86::COND_NE; 2125 case X86::JL_1: return X86::COND_L; 2126 case X86::JLE_1: return X86::COND_LE; 2127 case X86::JG_1: return X86::COND_G; 2128 case X86::JGE_1: return X86::COND_GE; 2129 case X86::JB_1: return X86::COND_B; 2130 case X86::JBE_1: return X86::COND_BE; 2131 case X86::JA_1: return X86::COND_A; 2132 case X86::JAE_1: return X86::COND_AE; 2133 case X86::JS_1: return X86::COND_S; 2134 case X86::JNS_1: return X86::COND_NS; 2135 case X86::JP_1: return X86::COND_P; 2136 case X86::JNP_1: return X86::COND_NP; 2137 case X86::JO_1: return X86::COND_O; 2138 case X86::JNO_1: return X86::COND_NO; 2139 } 2140 } 2141 2142 /// Return condition code of a SET opcode. 2143 X86::CondCode X86::getCondFromSETOpc(unsigned Opc) { 2144 switch (Opc) { 2145 default: return X86::COND_INVALID; 2146 case X86::SETAr: case X86::SETAm: return X86::COND_A; 2147 case X86::SETAEr: case X86::SETAEm: return X86::COND_AE; 2148 case X86::SETBr: case X86::SETBm: return X86::COND_B; 2149 case X86::SETBEr: case X86::SETBEm: return X86::COND_BE; 2150 case X86::SETEr: case X86::SETEm: return X86::COND_E; 2151 case X86::SETGr: case X86::SETGm: return X86::COND_G; 2152 case X86::SETGEr: case X86::SETGEm: return X86::COND_GE; 2153 case X86::SETLr: case X86::SETLm: return X86::COND_L; 2154 case X86::SETLEr: case X86::SETLEm: return X86::COND_LE; 2155 case X86::SETNEr: case X86::SETNEm: return X86::COND_NE; 2156 case X86::SETNOr: case X86::SETNOm: return X86::COND_NO; 2157 case X86::SETNPr: case X86::SETNPm: return X86::COND_NP; 2158 case X86::SETNSr: case X86::SETNSm: return X86::COND_NS; 2159 case X86::SETOr: case X86::SETOm: return X86::COND_O; 2160 case X86::SETPr: case X86::SETPm: return X86::COND_P; 2161 case X86::SETSr: case X86::SETSm: return X86::COND_S; 2162 } 2163 } 2164 2165 /// Return condition code of a CMov opcode. 2166 X86::CondCode X86::getCondFromCMovOpc(unsigned Opc) { 2167 switch (Opc) { 2168 default: return X86::COND_INVALID; 2169 case X86::CMOVA16rm: case X86::CMOVA16rr: case X86::CMOVA32rm: 2170 case X86::CMOVA32rr: case X86::CMOVA64rm: case X86::CMOVA64rr: 2171 return X86::COND_A; 2172 case X86::CMOVAE16rm: case X86::CMOVAE16rr: case X86::CMOVAE32rm: 2173 case X86::CMOVAE32rr: case X86::CMOVAE64rm: case X86::CMOVAE64rr: 2174 return X86::COND_AE; 2175 case X86::CMOVB16rm: case X86::CMOVB16rr: case X86::CMOVB32rm: 2176 case X86::CMOVB32rr: case X86::CMOVB64rm: case X86::CMOVB64rr: 2177 return X86::COND_B; 2178 case X86::CMOVBE16rm: case X86::CMOVBE16rr: case X86::CMOVBE32rm: 2179 case X86::CMOVBE32rr: case X86::CMOVBE64rm: case X86::CMOVBE64rr: 2180 return X86::COND_BE; 2181 case X86::CMOVE16rm: case X86::CMOVE16rr: case X86::CMOVE32rm: 2182 case X86::CMOVE32rr: case X86::CMOVE64rm: case X86::CMOVE64rr: 2183 return X86::COND_E; 2184 case X86::CMOVG16rm: case X86::CMOVG16rr: case X86::CMOVG32rm: 2185 case X86::CMOVG32rr: case X86::CMOVG64rm: case X86::CMOVG64rr: 2186 return X86::COND_G; 2187 case X86::CMOVGE16rm: case X86::CMOVGE16rr: case X86::CMOVGE32rm: 2188 case X86::CMOVGE32rr: case X86::CMOVGE64rm: case X86::CMOVGE64rr: 2189 return X86::COND_GE; 2190 case X86::CMOVL16rm: case X86::CMOVL16rr: case X86::CMOVL32rm: 2191 case X86::CMOVL32rr: case X86::CMOVL64rm: case X86::CMOVL64rr: 2192 return X86::COND_L; 2193 case X86::CMOVLE16rm: case X86::CMOVLE16rr: case X86::CMOVLE32rm: 2194 case X86::CMOVLE32rr: case X86::CMOVLE64rm: case X86::CMOVLE64rr: 2195 return X86::COND_LE; 2196 case X86::CMOVNE16rm: case X86::CMOVNE16rr: case X86::CMOVNE32rm: 2197 case X86::CMOVNE32rr: case X86::CMOVNE64rm: case X86::CMOVNE64rr: 2198 return X86::COND_NE; 2199 case X86::CMOVNO16rm: case X86::CMOVNO16rr: case X86::CMOVNO32rm: 2200 case X86::CMOVNO32rr: case X86::CMOVNO64rm: case X86::CMOVNO64rr: 2201 return X86::COND_NO; 2202 case X86::CMOVNP16rm: case X86::CMOVNP16rr: case X86::CMOVNP32rm: 2203 case X86::CMOVNP32rr: case X86::CMOVNP64rm: case X86::CMOVNP64rr: 2204 return X86::COND_NP; 2205 case X86::CMOVNS16rm: case X86::CMOVNS16rr: case X86::CMOVNS32rm: 2206 case X86::CMOVNS32rr: case X86::CMOVNS64rm: case X86::CMOVNS64rr: 2207 return X86::COND_NS; 2208 case X86::CMOVO16rm: case X86::CMOVO16rr: case X86::CMOVO32rm: 2209 case X86::CMOVO32rr: case X86::CMOVO64rm: case X86::CMOVO64rr: 2210 return X86::COND_O; 2211 case X86::CMOVP16rm: case X86::CMOVP16rr: case X86::CMOVP32rm: 2212 case X86::CMOVP32rr: case X86::CMOVP64rm: case X86::CMOVP64rr: 2213 return X86::COND_P; 2214 case X86::CMOVS16rm: case X86::CMOVS16rr: case X86::CMOVS32rm: 2215 case X86::CMOVS32rr: case X86::CMOVS64rm: case X86::CMOVS64rr: 2216 return X86::COND_S; 2217 } 2218 } 2219 2220 unsigned X86::GetCondBranchFromCond(X86::CondCode CC) { 2221 switch (CC) { 2222 default: llvm_unreachable("Illegal condition code!"); 2223 case X86::COND_E: return X86::JE_1; 2224 case X86::COND_NE: return X86::JNE_1; 2225 case X86::COND_L: return X86::JL_1; 2226 case X86::COND_LE: return X86::JLE_1; 2227 case X86::COND_G: return X86::JG_1; 2228 case X86::COND_GE: return X86::JGE_1; 2229 case X86::COND_B: return X86::JB_1; 2230 case X86::COND_BE: return X86::JBE_1; 2231 case X86::COND_A: return X86::JA_1; 2232 case X86::COND_AE: return X86::JAE_1; 2233 case X86::COND_S: return X86::JS_1; 2234 case X86::COND_NS: return X86::JNS_1; 2235 case X86::COND_P: return X86::JP_1; 2236 case X86::COND_NP: return X86::JNP_1; 2237 case X86::COND_O: return X86::JO_1; 2238 case X86::COND_NO: return X86::JNO_1; 2239 } 2240 } 2241 2242 /// Return the inverse of the specified condition, 2243 /// e.g. turning COND_E to COND_NE. 2244 X86::CondCode X86::GetOppositeBranchCondition(X86::CondCode CC) { 2245 switch (CC) { 2246 default: llvm_unreachable("Illegal condition code!"); 2247 case X86::COND_E: return X86::COND_NE; 2248 case X86::COND_NE: return X86::COND_E; 2249 case X86::COND_L: return X86::COND_GE; 2250 case X86::COND_LE: return X86::COND_G; 2251 case X86::COND_G: return X86::COND_LE; 2252 case X86::COND_GE: return X86::COND_L; 2253 case X86::COND_B: return X86::COND_AE; 2254 case X86::COND_BE: return X86::COND_A; 2255 case X86::COND_A: return X86::COND_BE; 2256 case X86::COND_AE: return X86::COND_B; 2257 case X86::COND_S: return X86::COND_NS; 2258 case X86::COND_NS: return X86::COND_S; 2259 case X86::COND_P: return X86::COND_NP; 2260 case X86::COND_NP: return X86::COND_P; 2261 case X86::COND_O: return X86::COND_NO; 2262 case X86::COND_NO: return X86::COND_O; 2263 case X86::COND_NE_OR_P: return X86::COND_E_AND_NP; 2264 case X86::COND_E_AND_NP: return X86::COND_NE_OR_P; 2265 } 2266 } 2267 2268 /// Assuming the flags are set by MI(a,b), return the condition code if we 2269 /// modify the instructions such that flags are set by MI(b,a). 2270 static X86::CondCode getSwappedCondition(X86::CondCode CC) { 2271 switch (CC) { 2272 default: return X86::COND_INVALID; 2273 case X86::COND_E: return X86::COND_E; 2274 case X86::COND_NE: return X86::COND_NE; 2275 case X86::COND_L: return X86::COND_G; 2276 case X86::COND_LE: return X86::COND_GE; 2277 case X86::COND_G: return X86::COND_L; 2278 case X86::COND_GE: return X86::COND_LE; 2279 case X86::COND_B: return X86::COND_A; 2280 case X86::COND_BE: return X86::COND_AE; 2281 case X86::COND_A: return X86::COND_B; 2282 case X86::COND_AE: return X86::COND_BE; 2283 } 2284 } 2285 2286 std::pair<X86::CondCode, bool> 2287 X86::getX86ConditionCode(CmpInst::Predicate Predicate) { 2288 X86::CondCode CC = X86::COND_INVALID; 2289 bool NeedSwap = false; 2290 switch (Predicate) { 2291 default: break; 2292 // Floating-point Predicates 2293 case CmpInst::FCMP_UEQ: CC = X86::COND_E; break; 2294 case CmpInst::FCMP_OLT: NeedSwap = true; LLVM_FALLTHROUGH; 2295 case CmpInst::FCMP_OGT: CC = X86::COND_A; break; 2296 case CmpInst::FCMP_OLE: NeedSwap = true; LLVM_FALLTHROUGH; 2297 case CmpInst::FCMP_OGE: CC = X86::COND_AE; break; 2298 case CmpInst::FCMP_UGT: NeedSwap = true; LLVM_FALLTHROUGH; 2299 case CmpInst::FCMP_ULT: CC = X86::COND_B; break; 2300 case CmpInst::FCMP_UGE: NeedSwap = true; LLVM_FALLTHROUGH; 2301 case CmpInst::FCMP_ULE: CC = X86::COND_BE; break; 2302 case CmpInst::FCMP_ONE: CC = X86::COND_NE; break; 2303 case CmpInst::FCMP_UNO: CC = X86::COND_P; break; 2304 case CmpInst::FCMP_ORD: CC = X86::COND_NP; break; 2305 case CmpInst::FCMP_OEQ: LLVM_FALLTHROUGH; 2306 case CmpInst::FCMP_UNE: CC = X86::COND_INVALID; break; 2307 2308 // Integer Predicates 2309 case CmpInst::ICMP_EQ: CC = X86::COND_E; break; 2310 case CmpInst::ICMP_NE: CC = X86::COND_NE; break; 2311 case CmpInst::ICMP_UGT: CC = X86::COND_A; break; 2312 case CmpInst::ICMP_UGE: CC = X86::COND_AE; break; 2313 case CmpInst::ICMP_ULT: CC = X86::COND_B; break; 2314 case CmpInst::ICMP_ULE: CC = X86::COND_BE; break; 2315 case CmpInst::ICMP_SGT: CC = X86::COND_G; break; 2316 case CmpInst::ICMP_SGE: CC = X86::COND_GE; break; 2317 case CmpInst::ICMP_SLT: CC = X86::COND_L; break; 2318 case CmpInst::ICMP_SLE: CC = X86::COND_LE; break; 2319 } 2320 2321 return std::make_pair(CC, NeedSwap); 2322 } 2323 2324 /// Return a set opcode for the given condition and 2325 /// whether it has memory operand. 2326 unsigned X86::getSETFromCond(CondCode CC, bool HasMemoryOperand) { 2327 static const uint16_t Opc[16][2] = { 2328 { X86::SETAr, X86::SETAm }, 2329 { X86::SETAEr, X86::SETAEm }, 2330 { X86::SETBr, X86::SETBm }, 2331 { X86::SETBEr, X86::SETBEm }, 2332 { X86::SETEr, X86::SETEm }, 2333 { X86::SETGr, X86::SETGm }, 2334 { X86::SETGEr, X86::SETGEm }, 2335 { X86::SETLr, X86::SETLm }, 2336 { X86::SETLEr, X86::SETLEm }, 2337 { X86::SETNEr, X86::SETNEm }, 2338 { X86::SETNOr, X86::SETNOm }, 2339 { X86::SETNPr, X86::SETNPm }, 2340 { X86::SETNSr, X86::SETNSm }, 2341 { X86::SETOr, X86::SETOm }, 2342 { X86::SETPr, X86::SETPm }, 2343 { X86::SETSr, X86::SETSm } 2344 }; 2345 2346 assert(CC <= LAST_VALID_COND && "Can only handle standard cond codes"); 2347 return Opc[CC][HasMemoryOperand ? 1 : 0]; 2348 } 2349 2350 /// Return a cmov opcode for the given condition, 2351 /// register size in bytes, and operand type. 2352 unsigned X86::getCMovFromCond(CondCode CC, unsigned RegBytes, 2353 bool HasMemoryOperand) { 2354 static const uint16_t Opc[32][3] = { 2355 { X86::CMOVA16rr, X86::CMOVA32rr, X86::CMOVA64rr }, 2356 { X86::CMOVAE16rr, X86::CMOVAE32rr, X86::CMOVAE64rr }, 2357 { X86::CMOVB16rr, X86::CMOVB32rr, X86::CMOVB64rr }, 2358 { X86::CMOVBE16rr, X86::CMOVBE32rr, X86::CMOVBE64rr }, 2359 { X86::CMOVE16rr, X86::CMOVE32rr, X86::CMOVE64rr }, 2360 { X86::CMOVG16rr, X86::CMOVG32rr, X86::CMOVG64rr }, 2361 { X86::CMOVGE16rr, X86::CMOVGE32rr, X86::CMOVGE64rr }, 2362 { X86::CMOVL16rr, X86::CMOVL32rr, X86::CMOVL64rr }, 2363 { X86::CMOVLE16rr, X86::CMOVLE32rr, X86::CMOVLE64rr }, 2364 { X86::CMOVNE16rr, X86::CMOVNE32rr, X86::CMOVNE64rr }, 2365 { X86::CMOVNO16rr, X86::CMOVNO32rr, X86::CMOVNO64rr }, 2366 { X86::CMOVNP16rr, X86::CMOVNP32rr, X86::CMOVNP64rr }, 2367 { X86::CMOVNS16rr, X86::CMOVNS32rr, X86::CMOVNS64rr }, 2368 { X86::CMOVO16rr, X86::CMOVO32rr, X86::CMOVO64rr }, 2369 { X86::CMOVP16rr, X86::CMOVP32rr, X86::CMOVP64rr }, 2370 { X86::CMOVS16rr, X86::CMOVS32rr, X86::CMOVS64rr }, 2371 { X86::CMOVA16rm, X86::CMOVA32rm, X86::CMOVA64rm }, 2372 { X86::CMOVAE16rm, X86::CMOVAE32rm, X86::CMOVAE64rm }, 2373 { X86::CMOVB16rm, X86::CMOVB32rm, X86::CMOVB64rm }, 2374 { X86::CMOVBE16rm, X86::CMOVBE32rm, X86::CMOVBE64rm }, 2375 { X86::CMOVE16rm, X86::CMOVE32rm, X86::CMOVE64rm }, 2376 { X86::CMOVG16rm, X86::CMOVG32rm, X86::CMOVG64rm }, 2377 { X86::CMOVGE16rm, X86::CMOVGE32rm, X86::CMOVGE64rm }, 2378 { X86::CMOVL16rm, X86::CMOVL32rm, X86::CMOVL64rm }, 2379 { X86::CMOVLE16rm, X86::CMOVLE32rm, X86::CMOVLE64rm }, 2380 { X86::CMOVNE16rm, X86::CMOVNE32rm, X86::CMOVNE64rm }, 2381 { X86::CMOVNO16rm, X86::CMOVNO32rm, X86::CMOVNO64rm }, 2382 { X86::CMOVNP16rm, X86::CMOVNP32rm, X86::CMOVNP64rm }, 2383 { X86::CMOVNS16rm, X86::CMOVNS32rm, X86::CMOVNS64rm }, 2384 { X86::CMOVO16rm, X86::CMOVO32rm, X86::CMOVO64rm }, 2385 { X86::CMOVP16rm, X86::CMOVP32rm, X86::CMOVP64rm }, 2386 { X86::CMOVS16rm, X86::CMOVS32rm, X86::CMOVS64rm } 2387 }; 2388 2389 assert(CC < 16 && "Can only handle standard cond codes"); 2390 unsigned Idx = HasMemoryOperand ? 16+CC : CC; 2391 switch(RegBytes) { 2392 default: llvm_unreachable("Illegal register size!"); 2393 case 2: return Opc[Idx][0]; 2394 case 4: return Opc[Idx][1]; 2395 case 8: return Opc[Idx][2]; 2396 } 2397 } 2398 2399 /// Get the VPCMP immediate for the given condition. 2400 unsigned X86::getVPCMPImmForCond(ISD::CondCode CC) { 2401 switch (CC) { 2402 default: llvm_unreachable("Unexpected SETCC condition"); 2403 case ISD::SETNE: return 4; 2404 case ISD::SETEQ: return 0; 2405 case ISD::SETULT: 2406 case ISD::SETLT: return 1; 2407 case ISD::SETUGT: 2408 case ISD::SETGT: return 6; 2409 case ISD::SETUGE: 2410 case ISD::SETGE: return 5; 2411 case ISD::SETULE: 2412 case ISD::SETLE: return 2; 2413 } 2414 } 2415 2416 /// Get the VPCMP immediate if the opcodes are swapped. 2417 unsigned X86::getSwappedVPCMPImm(unsigned Imm) { 2418 switch (Imm) { 2419 default: llvm_unreachable("Unreachable!"); 2420 case 0x01: Imm = 0x06; break; // LT -> NLE 2421 case 0x02: Imm = 0x05; break; // LE -> NLT 2422 case 0x05: Imm = 0x02; break; // NLT -> LE 2423 case 0x06: Imm = 0x01; break; // NLE -> LT 2424 case 0x00: // EQ 2425 case 0x03: // FALSE 2426 case 0x04: // NE 2427 case 0x07: // TRUE 2428 break; 2429 } 2430 2431 return Imm; 2432 } 2433 2434 /// Get the VPCOM immediate if the opcodes are swapped. 2435 unsigned X86::getSwappedVPCOMImm(unsigned Imm) { 2436 switch (Imm) { 2437 default: llvm_unreachable("Unreachable!"); 2438 case 0x00: Imm = 0x02; break; // LT -> GT 2439 case 0x01: Imm = 0x03; break; // LE -> GE 2440 case 0x02: Imm = 0x00; break; // GT -> LT 2441 case 0x03: Imm = 0x01; break; // GE -> LE 2442 case 0x04: // EQ 2443 case 0x05: // NE 2444 case 0x06: // FALSE 2445 case 0x07: // TRUE 2446 break; 2447 } 2448 2449 return Imm; 2450 } 2451 2452 bool X86InstrInfo::isUnpredicatedTerminator(const MachineInstr &MI) const { 2453 if (!MI.isTerminator()) return false; 2454 2455 // Conditional branch is a special case. 2456 if (MI.isBranch() && !MI.isBarrier()) 2457 return true; 2458 if (!MI.isPredicable()) 2459 return true; 2460 return !isPredicated(MI); 2461 } 2462 2463 bool X86InstrInfo::isUnconditionalTailCall(const MachineInstr &MI) const { 2464 switch (MI.getOpcode()) { 2465 case X86::TCRETURNdi: 2466 case X86::TCRETURNri: 2467 case X86::TCRETURNmi: 2468 case X86::TCRETURNdi64: 2469 case X86::TCRETURNri64: 2470 case X86::TCRETURNmi64: 2471 return true; 2472 default: 2473 return false; 2474 } 2475 } 2476 2477 bool X86InstrInfo::canMakeTailCallConditional( 2478 SmallVectorImpl<MachineOperand> &BranchCond, 2479 const MachineInstr &TailCall) const { 2480 if (TailCall.getOpcode() != X86::TCRETURNdi && 2481 TailCall.getOpcode() != X86::TCRETURNdi64) { 2482 // Only direct calls can be done with a conditional branch. 2483 return false; 2484 } 2485 2486 const MachineFunction *MF = TailCall.getParent()->getParent(); 2487 if (Subtarget.isTargetWin64() && MF->hasWinCFI()) { 2488 // Conditional tail calls confuse the Win64 unwinder. 2489 return false; 2490 } 2491 2492 assert(BranchCond.size() == 1); 2493 if (BranchCond[0].getImm() > X86::LAST_VALID_COND) { 2494 // Can't make a conditional tail call with this condition. 2495 return false; 2496 } 2497 2498 const X86MachineFunctionInfo *X86FI = MF->getInfo<X86MachineFunctionInfo>(); 2499 if (X86FI->getTCReturnAddrDelta() != 0 || 2500 TailCall.getOperand(1).getImm() != 0) { 2501 // A conditional tail call cannot do any stack adjustment. 2502 return false; 2503 } 2504 2505 return true; 2506 } 2507 2508 void X86InstrInfo::replaceBranchWithTailCall( 2509 MachineBasicBlock &MBB, SmallVectorImpl<MachineOperand> &BranchCond, 2510 const MachineInstr &TailCall) const { 2511 assert(canMakeTailCallConditional(BranchCond, TailCall)); 2512 2513 MachineBasicBlock::iterator I = MBB.end(); 2514 while (I != MBB.begin()) { 2515 --I; 2516 if (I->isDebugInstr()) 2517 continue; 2518 if (!I->isBranch()) 2519 assert(0 && "Can't find the branch to replace!"); 2520 2521 X86::CondCode CC = X86::getCondFromBranchOpc(I->getOpcode()); 2522 assert(BranchCond.size() == 1); 2523 if (CC != BranchCond[0].getImm()) 2524 continue; 2525 2526 break; 2527 } 2528 2529 unsigned Opc = TailCall.getOpcode() == X86::TCRETURNdi ? X86::TCRETURNdicc 2530 : X86::TCRETURNdi64cc; 2531 2532 auto MIB = BuildMI(MBB, I, MBB.findDebugLoc(I), get(Opc)); 2533 MIB->addOperand(TailCall.getOperand(0)); // Destination. 2534 MIB.addImm(0); // Stack offset (not used). 2535 MIB->addOperand(BranchCond[0]); // Condition. 2536 MIB.copyImplicitOps(TailCall); // Regmask and (imp-used) parameters. 2537 2538 // Add implicit uses and defs of all live regs potentially clobbered by the 2539 // call. This way they still appear live across the call. 2540 LivePhysRegs LiveRegs(getRegisterInfo()); 2541 LiveRegs.addLiveOuts(MBB); 2542 SmallVector<std::pair<unsigned, const MachineOperand *>, 8> Clobbers; 2543 LiveRegs.stepForward(*MIB, Clobbers); 2544 for (const auto &C : Clobbers) { 2545 MIB.addReg(C.first, RegState::Implicit); 2546 MIB.addReg(C.first, RegState::Implicit | RegState::Define); 2547 } 2548 2549 I->eraseFromParent(); 2550 } 2551 2552 // Given a MBB and its TBB, find the FBB which was a fallthrough MBB (it may 2553 // not be a fallthrough MBB now due to layout changes). Return nullptr if the 2554 // fallthrough MBB cannot be identified. 2555 static MachineBasicBlock *getFallThroughMBB(MachineBasicBlock *MBB, 2556 MachineBasicBlock *TBB) { 2557 // Look for non-EHPad successors other than TBB. If we find exactly one, it 2558 // is the fallthrough MBB. If we find zero, then TBB is both the target MBB 2559 // and fallthrough MBB. If we find more than one, we cannot identify the 2560 // fallthrough MBB and should return nullptr. 2561 MachineBasicBlock *FallthroughBB = nullptr; 2562 for (auto SI = MBB->succ_begin(), SE = MBB->succ_end(); SI != SE; ++SI) { 2563 if ((*SI)->isEHPad() || (*SI == TBB && FallthroughBB)) 2564 continue; 2565 // Return a nullptr if we found more than one fallthrough successor. 2566 if (FallthroughBB && FallthroughBB != TBB) 2567 return nullptr; 2568 FallthroughBB = *SI; 2569 } 2570 return FallthroughBB; 2571 } 2572 2573 bool X86InstrInfo::AnalyzeBranchImpl( 2574 MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, 2575 SmallVectorImpl<MachineOperand> &Cond, 2576 SmallVectorImpl<MachineInstr *> &CondBranches, bool AllowModify) const { 2577 2578 // Start from the bottom of the block and work up, examining the 2579 // terminator instructions. 2580 MachineBasicBlock::iterator I = MBB.end(); 2581 MachineBasicBlock::iterator UnCondBrIter = MBB.end(); 2582 while (I != MBB.begin()) { 2583 --I; 2584 if (I->isDebugInstr()) 2585 continue; 2586 2587 // Working from the bottom, when we see a non-terminator instruction, we're 2588 // done. 2589 if (!isUnpredicatedTerminator(*I)) 2590 break; 2591 2592 // A terminator that isn't a branch can't easily be handled by this 2593 // analysis. 2594 if (!I->isBranch()) 2595 return true; 2596 2597 // Handle unconditional branches. 2598 if (I->getOpcode() == X86::JMP_1) { 2599 UnCondBrIter = I; 2600 2601 if (!AllowModify) { 2602 TBB = I->getOperand(0).getMBB(); 2603 continue; 2604 } 2605 2606 // If the block has any instructions after a JMP, delete them. 2607 while (std::next(I) != MBB.end()) 2608 std::next(I)->eraseFromParent(); 2609 2610 Cond.clear(); 2611 FBB = nullptr; 2612 2613 // Delete the JMP if it's equivalent to a fall-through. 2614 if (MBB.isLayoutSuccessor(I->getOperand(0).getMBB())) { 2615 TBB = nullptr; 2616 I->eraseFromParent(); 2617 I = MBB.end(); 2618 UnCondBrIter = MBB.end(); 2619 continue; 2620 } 2621 2622 // TBB is used to indicate the unconditional destination. 2623 TBB = I->getOperand(0).getMBB(); 2624 continue; 2625 } 2626 2627 // Handle conditional branches. 2628 X86::CondCode BranchCode = X86::getCondFromBranchOpc(I->getOpcode()); 2629 if (BranchCode == X86::COND_INVALID) 2630 return true; // Can't handle indirect branch. 2631 2632 // Working from the bottom, handle the first conditional branch. 2633 if (Cond.empty()) { 2634 MachineBasicBlock *TargetBB = I->getOperand(0).getMBB(); 2635 if (AllowModify && UnCondBrIter != MBB.end() && 2636 MBB.isLayoutSuccessor(TargetBB)) { 2637 // If we can modify the code and it ends in something like: 2638 // 2639 // jCC L1 2640 // jmp L2 2641 // L1: 2642 // ... 2643 // L2: 2644 // 2645 // Then we can change this to: 2646 // 2647 // jnCC L2 2648 // L1: 2649 // ... 2650 // L2: 2651 // 2652 // Which is a bit more efficient. 2653 // We conditionally jump to the fall-through block. 2654 BranchCode = GetOppositeBranchCondition(BranchCode); 2655 unsigned JNCC = GetCondBranchFromCond(BranchCode); 2656 MachineBasicBlock::iterator OldInst = I; 2657 2658 BuildMI(MBB, UnCondBrIter, MBB.findDebugLoc(I), get(JNCC)) 2659 .addMBB(UnCondBrIter->getOperand(0).getMBB()); 2660 BuildMI(MBB, UnCondBrIter, MBB.findDebugLoc(I), get(X86::JMP_1)) 2661 .addMBB(TargetBB); 2662 2663 OldInst->eraseFromParent(); 2664 UnCondBrIter->eraseFromParent(); 2665 2666 // Restart the analysis. 2667 UnCondBrIter = MBB.end(); 2668 I = MBB.end(); 2669 continue; 2670 } 2671 2672 FBB = TBB; 2673 TBB = I->getOperand(0).getMBB(); 2674 Cond.push_back(MachineOperand::CreateImm(BranchCode)); 2675 CondBranches.push_back(&*I); 2676 continue; 2677 } 2678 2679 // Handle subsequent conditional branches. Only handle the case where all 2680 // conditional branches branch to the same destination and their condition 2681 // opcodes fit one of the special multi-branch idioms. 2682 assert(Cond.size() == 1); 2683 assert(TBB); 2684 2685 // If the conditions are the same, we can leave them alone. 2686 X86::CondCode OldBranchCode = (X86::CondCode)Cond[0].getImm(); 2687 auto NewTBB = I->getOperand(0).getMBB(); 2688 if (OldBranchCode == BranchCode && TBB == NewTBB) 2689 continue; 2690 2691 // If they differ, see if they fit one of the known patterns. Theoretically, 2692 // we could handle more patterns here, but we shouldn't expect to see them 2693 // if instruction selection has done a reasonable job. 2694 if (TBB == NewTBB && 2695 ((OldBranchCode == X86::COND_P && BranchCode == X86::COND_NE) || 2696 (OldBranchCode == X86::COND_NE && BranchCode == X86::COND_P))) { 2697 BranchCode = X86::COND_NE_OR_P; 2698 } else if ((OldBranchCode == X86::COND_NP && BranchCode == X86::COND_NE) || 2699 (OldBranchCode == X86::COND_E && BranchCode == X86::COND_P)) { 2700 if (NewTBB != (FBB ? FBB : getFallThroughMBB(&MBB, TBB))) 2701 return true; 2702 2703 // X86::COND_E_AND_NP usually has two different branch destinations. 2704 // 2705 // JP B1 2706 // JE B2 2707 // JMP B1 2708 // B1: 2709 // B2: 2710 // 2711 // Here this condition branches to B2 only if NP && E. It has another 2712 // equivalent form: 2713 // 2714 // JNE B1 2715 // JNP B2 2716 // JMP B1 2717 // B1: 2718 // B2: 2719 // 2720 // Similarly it branches to B2 only if E && NP. That is why this condition 2721 // is named with COND_E_AND_NP. 2722 BranchCode = X86::COND_E_AND_NP; 2723 } else 2724 return true; 2725 2726 // Update the MachineOperand. 2727 Cond[0].setImm(BranchCode); 2728 CondBranches.push_back(&*I); 2729 } 2730 2731 return false; 2732 } 2733 2734 bool X86InstrInfo::analyzeBranch(MachineBasicBlock &MBB, 2735 MachineBasicBlock *&TBB, 2736 MachineBasicBlock *&FBB, 2737 SmallVectorImpl<MachineOperand> &Cond, 2738 bool AllowModify) const { 2739 SmallVector<MachineInstr *, 4> CondBranches; 2740 return AnalyzeBranchImpl(MBB, TBB, FBB, Cond, CondBranches, AllowModify); 2741 } 2742 2743 bool X86InstrInfo::analyzeBranchPredicate(MachineBasicBlock &MBB, 2744 MachineBranchPredicate &MBP, 2745 bool AllowModify) const { 2746 using namespace std::placeholders; 2747 2748 SmallVector<MachineOperand, 4> Cond; 2749 SmallVector<MachineInstr *, 4> CondBranches; 2750 if (AnalyzeBranchImpl(MBB, MBP.TrueDest, MBP.FalseDest, Cond, CondBranches, 2751 AllowModify)) 2752 return true; 2753 2754 if (Cond.size() != 1) 2755 return true; 2756 2757 assert(MBP.TrueDest && "expected!"); 2758 2759 if (!MBP.FalseDest) 2760 MBP.FalseDest = MBB.getNextNode(); 2761 2762 const TargetRegisterInfo *TRI = &getRegisterInfo(); 2763 2764 MachineInstr *ConditionDef = nullptr; 2765 bool SingleUseCondition = true; 2766 2767 for (auto I = std::next(MBB.rbegin()), E = MBB.rend(); I != E; ++I) { 2768 if (I->modifiesRegister(X86::EFLAGS, TRI)) { 2769 ConditionDef = &*I; 2770 break; 2771 } 2772 2773 if (I->readsRegister(X86::EFLAGS, TRI)) 2774 SingleUseCondition = false; 2775 } 2776 2777 if (!ConditionDef) 2778 return true; 2779 2780 if (SingleUseCondition) { 2781 for (auto *Succ : MBB.successors()) 2782 if (Succ->isLiveIn(X86::EFLAGS)) 2783 SingleUseCondition = false; 2784 } 2785 2786 MBP.ConditionDef = ConditionDef; 2787 MBP.SingleUseCondition = SingleUseCondition; 2788 2789 // Currently we only recognize the simple pattern: 2790 // 2791 // test %reg, %reg 2792 // je %label 2793 // 2794 const unsigned TestOpcode = 2795 Subtarget.is64Bit() ? X86::TEST64rr : X86::TEST32rr; 2796 2797 if (ConditionDef->getOpcode() == TestOpcode && 2798 ConditionDef->getNumOperands() == 3 && 2799 ConditionDef->getOperand(0).isIdenticalTo(ConditionDef->getOperand(1)) && 2800 (Cond[0].getImm() == X86::COND_NE || Cond[0].getImm() == X86::COND_E)) { 2801 MBP.LHS = ConditionDef->getOperand(0); 2802 MBP.RHS = MachineOperand::CreateImm(0); 2803 MBP.Predicate = Cond[0].getImm() == X86::COND_NE 2804 ? MachineBranchPredicate::PRED_NE 2805 : MachineBranchPredicate::PRED_EQ; 2806 return false; 2807 } 2808 2809 return true; 2810 } 2811 2812 unsigned X86InstrInfo::removeBranch(MachineBasicBlock &MBB, 2813 int *BytesRemoved) const { 2814 assert(!BytesRemoved && "code size not handled"); 2815 2816 MachineBasicBlock::iterator I = MBB.end(); 2817 unsigned Count = 0; 2818 2819 while (I != MBB.begin()) { 2820 --I; 2821 if (I->isDebugInstr()) 2822 continue; 2823 if (I->getOpcode() != X86::JMP_1 && 2824 X86::getCondFromBranchOpc(I->getOpcode()) == X86::COND_INVALID) 2825 break; 2826 // Remove the branch. 2827 I->eraseFromParent(); 2828 I = MBB.end(); 2829 ++Count; 2830 } 2831 2832 return Count; 2833 } 2834 2835 unsigned X86InstrInfo::insertBranch(MachineBasicBlock &MBB, 2836 MachineBasicBlock *TBB, 2837 MachineBasicBlock *FBB, 2838 ArrayRef<MachineOperand> Cond, 2839 const DebugLoc &DL, 2840 int *BytesAdded) const { 2841 // Shouldn't be a fall through. 2842 assert(TBB && "insertBranch must not be told to insert a fallthrough"); 2843 assert((Cond.size() == 1 || Cond.size() == 0) && 2844 "X86 branch conditions have one component!"); 2845 assert(!BytesAdded && "code size not handled"); 2846 2847 if (Cond.empty()) { 2848 // Unconditional branch? 2849 assert(!FBB && "Unconditional branch with multiple successors!"); 2850 BuildMI(&MBB, DL, get(X86::JMP_1)).addMBB(TBB); 2851 return 1; 2852 } 2853 2854 // If FBB is null, it is implied to be a fall-through block. 2855 bool FallThru = FBB == nullptr; 2856 2857 // Conditional branch. 2858 unsigned Count = 0; 2859 X86::CondCode CC = (X86::CondCode)Cond[0].getImm(); 2860 switch (CC) { 2861 case X86::COND_NE_OR_P: 2862 // Synthesize NE_OR_P with two branches. 2863 BuildMI(&MBB, DL, get(X86::JNE_1)).addMBB(TBB); 2864 ++Count; 2865 BuildMI(&MBB, DL, get(X86::JP_1)).addMBB(TBB); 2866 ++Count; 2867 break; 2868 case X86::COND_E_AND_NP: 2869 // Use the next block of MBB as FBB if it is null. 2870 if (FBB == nullptr) { 2871 FBB = getFallThroughMBB(&MBB, TBB); 2872 assert(FBB && "MBB cannot be the last block in function when the false " 2873 "body is a fall-through."); 2874 } 2875 // Synthesize COND_E_AND_NP with two branches. 2876 BuildMI(&MBB, DL, get(X86::JNE_1)).addMBB(FBB); 2877 ++Count; 2878 BuildMI(&MBB, DL, get(X86::JNP_1)).addMBB(TBB); 2879 ++Count; 2880 break; 2881 default: { 2882 unsigned Opc = GetCondBranchFromCond(CC); 2883 BuildMI(&MBB, DL, get(Opc)).addMBB(TBB); 2884 ++Count; 2885 } 2886 } 2887 if (!FallThru) { 2888 // Two-way Conditional branch. Insert the second branch. 2889 BuildMI(&MBB, DL, get(X86::JMP_1)).addMBB(FBB); 2890 ++Count; 2891 } 2892 return Count; 2893 } 2894 2895 bool X86InstrInfo:: 2896 canInsertSelect(const MachineBasicBlock &MBB, 2897 ArrayRef<MachineOperand> Cond, 2898 unsigned TrueReg, unsigned FalseReg, 2899 int &CondCycles, int &TrueCycles, int &FalseCycles) const { 2900 // Not all subtargets have cmov instructions. 2901 if (!Subtarget.hasCMov()) 2902 return false; 2903 if (Cond.size() != 1) 2904 return false; 2905 // We cannot do the composite conditions, at least not in SSA form. 2906 if ((X86::CondCode)Cond[0].getImm() > X86::COND_S) 2907 return false; 2908 2909 // Check register classes. 2910 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 2911 const TargetRegisterClass *RC = 2912 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg)); 2913 if (!RC) 2914 return false; 2915 2916 // We have cmov instructions for 16, 32, and 64 bit general purpose registers. 2917 if (X86::GR16RegClass.hasSubClassEq(RC) || 2918 X86::GR32RegClass.hasSubClassEq(RC) || 2919 X86::GR64RegClass.hasSubClassEq(RC)) { 2920 // This latency applies to Pentium M, Merom, Wolfdale, Nehalem, and Sandy 2921 // Bridge. Probably Ivy Bridge as well. 2922 CondCycles = 2; 2923 TrueCycles = 2; 2924 FalseCycles = 2; 2925 return true; 2926 } 2927 2928 // Can't do vectors. 2929 return false; 2930 } 2931 2932 void X86InstrInfo::insertSelect(MachineBasicBlock &MBB, 2933 MachineBasicBlock::iterator I, 2934 const DebugLoc &DL, unsigned DstReg, 2935 ArrayRef<MachineOperand> Cond, unsigned TrueReg, 2936 unsigned FalseReg) const { 2937 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 2938 const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo(); 2939 const TargetRegisterClass &RC = *MRI.getRegClass(DstReg); 2940 assert(Cond.size() == 1 && "Invalid Cond array"); 2941 unsigned Opc = getCMovFromCond((X86::CondCode)Cond[0].getImm(), 2942 TRI.getRegSizeInBits(RC) / 8, 2943 false /*HasMemoryOperand*/); 2944 BuildMI(MBB, I, DL, get(Opc), DstReg).addReg(FalseReg).addReg(TrueReg); 2945 } 2946 2947 /// Test if the given register is a physical h register. 2948 static bool isHReg(unsigned Reg) { 2949 return X86::GR8_ABCD_HRegClass.contains(Reg); 2950 } 2951 2952 // Try and copy between VR128/VR64 and GR64 registers. 2953 static unsigned CopyToFromAsymmetricReg(unsigned DestReg, unsigned SrcReg, 2954 const X86Subtarget &Subtarget) { 2955 bool HasAVX = Subtarget.hasAVX(); 2956 bool HasAVX512 = Subtarget.hasAVX512(); 2957 2958 // SrcReg(MaskReg) -> DestReg(GR64) 2959 // SrcReg(MaskReg) -> DestReg(GR32) 2960 2961 // All KMASK RegClasses hold the same k registers, can be tested against anyone. 2962 if (X86::VK16RegClass.contains(SrcReg)) { 2963 if (X86::GR64RegClass.contains(DestReg)) { 2964 assert(Subtarget.hasBWI()); 2965 return X86::KMOVQrk; 2966 } 2967 if (X86::GR32RegClass.contains(DestReg)) 2968 return Subtarget.hasBWI() ? X86::KMOVDrk : X86::KMOVWrk; 2969 } 2970 2971 // SrcReg(GR64) -> DestReg(MaskReg) 2972 // SrcReg(GR32) -> DestReg(MaskReg) 2973 2974 // All KMASK RegClasses hold the same k registers, can be tested against anyone. 2975 if (X86::VK16RegClass.contains(DestReg)) { 2976 if (X86::GR64RegClass.contains(SrcReg)) { 2977 assert(Subtarget.hasBWI()); 2978 return X86::KMOVQkr; 2979 } 2980 if (X86::GR32RegClass.contains(SrcReg)) 2981 return Subtarget.hasBWI() ? X86::KMOVDkr : X86::KMOVWkr; 2982 } 2983 2984 2985 // SrcReg(VR128) -> DestReg(GR64) 2986 // SrcReg(VR64) -> DestReg(GR64) 2987 // SrcReg(GR64) -> DestReg(VR128) 2988 // SrcReg(GR64) -> DestReg(VR64) 2989 2990 if (X86::GR64RegClass.contains(DestReg)) { 2991 if (X86::VR128XRegClass.contains(SrcReg)) 2992 // Copy from a VR128 register to a GR64 register. 2993 return HasAVX512 ? X86::VMOVPQIto64Zrr : 2994 HasAVX ? X86::VMOVPQIto64rr : 2995 X86::MOVPQIto64rr; 2996 if (X86::VR64RegClass.contains(SrcReg)) 2997 // Copy from a VR64 register to a GR64 register. 2998 return X86::MMX_MOVD64from64rr; 2999 } else if (X86::GR64RegClass.contains(SrcReg)) { 3000 // Copy from a GR64 register to a VR128 register. 3001 if (X86::VR128XRegClass.contains(DestReg)) 3002 return HasAVX512 ? X86::VMOV64toPQIZrr : 3003 HasAVX ? X86::VMOV64toPQIrr : 3004 X86::MOV64toPQIrr; 3005 // Copy from a GR64 register to a VR64 register. 3006 if (X86::VR64RegClass.contains(DestReg)) 3007 return X86::MMX_MOVD64to64rr; 3008 } 3009 3010 // SrcReg(FR32) -> DestReg(GR32) 3011 // SrcReg(GR32) -> DestReg(FR32) 3012 3013 if (X86::GR32RegClass.contains(DestReg) && 3014 X86::FR32XRegClass.contains(SrcReg)) 3015 // Copy from a FR32 register to a GR32 register. 3016 return HasAVX512 ? X86::VMOVSS2DIZrr : 3017 HasAVX ? X86::VMOVSS2DIrr : 3018 X86::MOVSS2DIrr; 3019 3020 if (X86::FR32XRegClass.contains(DestReg) && 3021 X86::GR32RegClass.contains(SrcReg)) 3022 // Copy from a GR32 register to a FR32 register. 3023 return HasAVX512 ? X86::VMOVDI2SSZrr : 3024 HasAVX ? X86::VMOVDI2SSrr : 3025 X86::MOVDI2SSrr; 3026 return 0; 3027 } 3028 3029 void X86InstrInfo::copyPhysReg(MachineBasicBlock &MBB, 3030 MachineBasicBlock::iterator MI, 3031 const DebugLoc &DL, unsigned DestReg, 3032 unsigned SrcReg, bool KillSrc) const { 3033 // First deal with the normal symmetric copies. 3034 bool HasAVX = Subtarget.hasAVX(); 3035 bool HasVLX = Subtarget.hasVLX(); 3036 unsigned Opc = 0; 3037 if (X86::GR64RegClass.contains(DestReg, SrcReg)) 3038 Opc = X86::MOV64rr; 3039 else if (X86::GR32RegClass.contains(DestReg, SrcReg)) 3040 Opc = X86::MOV32rr; 3041 else if (X86::GR16RegClass.contains(DestReg, SrcReg)) 3042 Opc = X86::MOV16rr; 3043 else if (X86::GR8RegClass.contains(DestReg, SrcReg)) { 3044 // Copying to or from a physical H register on x86-64 requires a NOREX 3045 // move. Otherwise use a normal move. 3046 if ((isHReg(DestReg) || isHReg(SrcReg)) && 3047 Subtarget.is64Bit()) { 3048 Opc = X86::MOV8rr_NOREX; 3049 // Both operands must be encodable without an REX prefix. 3050 assert(X86::GR8_NOREXRegClass.contains(SrcReg, DestReg) && 3051 "8-bit H register can not be copied outside GR8_NOREX"); 3052 } else 3053 Opc = X86::MOV8rr; 3054 } 3055 else if (X86::VR64RegClass.contains(DestReg, SrcReg)) 3056 Opc = X86::MMX_MOVQ64rr; 3057 else if (X86::VR128XRegClass.contains(DestReg, SrcReg)) { 3058 if (HasVLX) 3059 Opc = X86::VMOVAPSZ128rr; 3060 else if (X86::VR128RegClass.contains(DestReg, SrcReg)) 3061 Opc = HasAVX ? X86::VMOVAPSrr : X86::MOVAPSrr; 3062 else { 3063 // If this an extended register and we don't have VLX we need to use a 3064 // 512-bit move. 3065 Opc = X86::VMOVAPSZrr; 3066 const TargetRegisterInfo *TRI = &getRegisterInfo(); 3067 DestReg = TRI->getMatchingSuperReg(DestReg, X86::sub_xmm, 3068 &X86::VR512RegClass); 3069 SrcReg = TRI->getMatchingSuperReg(SrcReg, X86::sub_xmm, 3070 &X86::VR512RegClass); 3071 } 3072 } else if (X86::VR256XRegClass.contains(DestReg, SrcReg)) { 3073 if (HasVLX) 3074 Opc = X86::VMOVAPSZ256rr; 3075 else if (X86::VR256RegClass.contains(DestReg, SrcReg)) 3076 Opc = X86::VMOVAPSYrr; 3077 else { 3078 // If this an extended register and we don't have VLX we need to use a 3079 // 512-bit move. 3080 Opc = X86::VMOVAPSZrr; 3081 const TargetRegisterInfo *TRI = &getRegisterInfo(); 3082 DestReg = TRI->getMatchingSuperReg(DestReg, X86::sub_ymm, 3083 &X86::VR512RegClass); 3084 SrcReg = TRI->getMatchingSuperReg(SrcReg, X86::sub_ymm, 3085 &X86::VR512RegClass); 3086 } 3087 } else if (X86::VR512RegClass.contains(DestReg, SrcReg)) 3088 Opc = X86::VMOVAPSZrr; 3089 // All KMASK RegClasses hold the same k registers, can be tested against anyone. 3090 else if (X86::VK16RegClass.contains(DestReg, SrcReg)) 3091 Opc = Subtarget.hasBWI() ? X86::KMOVQkk : X86::KMOVWkk; 3092 if (!Opc) 3093 Opc = CopyToFromAsymmetricReg(DestReg, SrcReg, Subtarget); 3094 3095 if (Opc) { 3096 BuildMI(MBB, MI, DL, get(Opc), DestReg) 3097 .addReg(SrcReg, getKillRegState(KillSrc)); 3098 return; 3099 } 3100 3101 if (SrcReg == X86::EFLAGS || DestReg == X86::EFLAGS) { 3102 // FIXME: We use a fatal error here because historically LLVM has tried 3103 // lower some of these physreg copies and we want to ensure we get 3104 // reasonable bug reports if someone encounters a case no other testing 3105 // found. This path should be removed after the LLVM 7 release. 3106 report_fatal_error("Unable to copy EFLAGS physical register!"); 3107 } 3108 3109 LLVM_DEBUG(dbgs() << "Cannot copy " << RI.getName(SrcReg) << " to " 3110 << RI.getName(DestReg) << '\n'); 3111 llvm_unreachable("Cannot emit physreg copy instruction"); 3112 } 3113 3114 bool X86InstrInfo::isCopyInstr(const MachineInstr &MI, 3115 const MachineOperand *&Src, 3116 const MachineOperand *&Dest) const { 3117 if (MI.isMoveReg()) { 3118 Dest = &MI.getOperand(0); 3119 Src = &MI.getOperand(1); 3120 return true; 3121 } 3122 return false; 3123 } 3124 3125 static unsigned getLoadStoreRegOpcode(unsigned Reg, 3126 const TargetRegisterClass *RC, 3127 bool isStackAligned, 3128 const X86Subtarget &STI, 3129 bool load) { 3130 bool HasAVX = STI.hasAVX(); 3131 bool HasAVX512 = STI.hasAVX512(); 3132 bool HasVLX = STI.hasVLX(); 3133 3134 switch (STI.getRegisterInfo()->getSpillSize(*RC)) { 3135 default: 3136 llvm_unreachable("Unknown spill size"); 3137 case 1: 3138 assert(X86::GR8RegClass.hasSubClassEq(RC) && "Unknown 1-byte regclass"); 3139 if (STI.is64Bit()) 3140 // Copying to or from a physical H register on x86-64 requires a NOREX 3141 // move. Otherwise use a normal move. 3142 if (isHReg(Reg) || X86::GR8_ABCD_HRegClass.hasSubClassEq(RC)) 3143 return load ? X86::MOV8rm_NOREX : X86::MOV8mr_NOREX; 3144 return load ? X86::MOV8rm : X86::MOV8mr; 3145 case 2: 3146 if (X86::VK16RegClass.hasSubClassEq(RC)) 3147 return load ? X86::KMOVWkm : X86::KMOVWmk; 3148 assert(X86::GR16RegClass.hasSubClassEq(RC) && "Unknown 2-byte regclass"); 3149 return load ? X86::MOV16rm : X86::MOV16mr; 3150 case 4: 3151 if (X86::GR32RegClass.hasSubClassEq(RC)) 3152 return load ? X86::MOV32rm : X86::MOV32mr; 3153 if (X86::FR32XRegClass.hasSubClassEq(RC)) 3154 return load ? 3155 (HasAVX512 ? X86::VMOVSSZrm : HasAVX ? X86::VMOVSSrm : X86::MOVSSrm) : 3156 (HasAVX512 ? X86::VMOVSSZmr : HasAVX ? X86::VMOVSSmr : X86::MOVSSmr); 3157 if (X86::RFP32RegClass.hasSubClassEq(RC)) 3158 return load ? X86::LD_Fp32m : X86::ST_Fp32m; 3159 if (X86::VK32RegClass.hasSubClassEq(RC)) { 3160 assert(STI.hasBWI() && "KMOVD requires BWI"); 3161 return load ? X86::KMOVDkm : X86::KMOVDmk; 3162 } 3163 llvm_unreachable("Unknown 4-byte regclass"); 3164 case 8: 3165 if (X86::GR64RegClass.hasSubClassEq(RC)) 3166 return load ? X86::MOV64rm : X86::MOV64mr; 3167 if (X86::FR64XRegClass.hasSubClassEq(RC)) 3168 return load ? 3169 (HasAVX512 ? X86::VMOVSDZrm : HasAVX ? X86::VMOVSDrm : X86::MOVSDrm) : 3170 (HasAVX512 ? X86::VMOVSDZmr : HasAVX ? X86::VMOVSDmr : X86::MOVSDmr); 3171 if (X86::VR64RegClass.hasSubClassEq(RC)) 3172 return load ? X86::MMX_MOVQ64rm : X86::MMX_MOVQ64mr; 3173 if (X86::RFP64RegClass.hasSubClassEq(RC)) 3174 return load ? X86::LD_Fp64m : X86::ST_Fp64m; 3175 if (X86::VK64RegClass.hasSubClassEq(RC)) { 3176 assert(STI.hasBWI() && "KMOVQ requires BWI"); 3177 return load ? X86::KMOVQkm : X86::KMOVQmk; 3178 } 3179 llvm_unreachable("Unknown 8-byte regclass"); 3180 case 10: 3181 assert(X86::RFP80RegClass.hasSubClassEq(RC) && "Unknown 10-byte regclass"); 3182 return load ? X86::LD_Fp80m : X86::ST_FpP80m; 3183 case 16: { 3184 if (X86::VR128XRegClass.hasSubClassEq(RC)) { 3185 // If stack is realigned we can use aligned stores. 3186 if (isStackAligned) 3187 return load ? 3188 (HasVLX ? X86::VMOVAPSZ128rm : 3189 HasAVX512 ? X86::VMOVAPSZ128rm_NOVLX : 3190 HasAVX ? X86::VMOVAPSrm : 3191 X86::MOVAPSrm): 3192 (HasVLX ? X86::VMOVAPSZ128mr : 3193 HasAVX512 ? X86::VMOVAPSZ128mr_NOVLX : 3194 HasAVX ? X86::VMOVAPSmr : 3195 X86::MOVAPSmr); 3196 else 3197 return load ? 3198 (HasVLX ? X86::VMOVUPSZ128rm : 3199 HasAVX512 ? X86::VMOVUPSZ128rm_NOVLX : 3200 HasAVX ? X86::VMOVUPSrm : 3201 X86::MOVUPSrm): 3202 (HasVLX ? X86::VMOVUPSZ128mr : 3203 HasAVX512 ? X86::VMOVUPSZ128mr_NOVLX : 3204 HasAVX ? X86::VMOVUPSmr : 3205 X86::MOVUPSmr); 3206 } 3207 if (X86::BNDRRegClass.hasSubClassEq(RC)) { 3208 if (STI.is64Bit()) 3209 return load ? X86::BNDMOV64rm : X86::BNDMOV64mr; 3210 else 3211 return load ? X86::BNDMOV32rm : X86::BNDMOV32mr; 3212 } 3213 llvm_unreachable("Unknown 16-byte regclass"); 3214 } 3215 case 32: 3216 assert(X86::VR256XRegClass.hasSubClassEq(RC) && "Unknown 32-byte regclass"); 3217 // If stack is realigned we can use aligned stores. 3218 if (isStackAligned) 3219 return load ? 3220 (HasVLX ? X86::VMOVAPSZ256rm : 3221 HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX : 3222 X86::VMOVAPSYrm) : 3223 (HasVLX ? X86::VMOVAPSZ256mr : 3224 HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX : 3225 X86::VMOVAPSYmr); 3226 else 3227 return load ? 3228 (HasVLX ? X86::VMOVUPSZ256rm : 3229 HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX : 3230 X86::VMOVUPSYrm) : 3231 (HasVLX ? X86::VMOVUPSZ256mr : 3232 HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX : 3233 X86::VMOVUPSYmr); 3234 case 64: 3235 assert(X86::VR512RegClass.hasSubClassEq(RC) && "Unknown 64-byte regclass"); 3236 assert(STI.hasAVX512() && "Using 512-bit register requires AVX512"); 3237 if (isStackAligned) 3238 return load ? X86::VMOVAPSZrm : X86::VMOVAPSZmr; 3239 else 3240 return load ? X86::VMOVUPSZrm : X86::VMOVUPSZmr; 3241 } 3242 } 3243 3244 bool X86InstrInfo::getMemOpBaseRegImmOfs(MachineInstr &MemOp, unsigned &BaseReg, 3245 int64_t &Offset, 3246 const TargetRegisterInfo *TRI) const { 3247 const MCInstrDesc &Desc = MemOp.getDesc(); 3248 int MemRefBegin = X86II::getMemoryOperandNo(Desc.TSFlags); 3249 if (MemRefBegin < 0) 3250 return false; 3251 3252 MemRefBegin += X86II::getOperandBias(Desc); 3253 3254 MachineOperand &BaseMO = MemOp.getOperand(MemRefBegin + X86::AddrBaseReg); 3255 if (!BaseMO.isReg()) // Can be an MO_FrameIndex 3256 return false; 3257 3258 BaseReg = BaseMO.getReg(); 3259 if (MemOp.getOperand(MemRefBegin + X86::AddrScaleAmt).getImm() != 1) 3260 return false; 3261 3262 if (MemOp.getOperand(MemRefBegin + X86::AddrIndexReg).getReg() != 3263 X86::NoRegister) 3264 return false; 3265 3266 const MachineOperand &DispMO = MemOp.getOperand(MemRefBegin + X86::AddrDisp); 3267 3268 // Displacement can be symbolic 3269 if (!DispMO.isImm()) 3270 return false; 3271 3272 Offset = DispMO.getImm(); 3273 3274 return true; 3275 } 3276 3277 static unsigned getStoreRegOpcode(unsigned SrcReg, 3278 const TargetRegisterClass *RC, 3279 bool isStackAligned, 3280 const X86Subtarget &STI) { 3281 return getLoadStoreRegOpcode(SrcReg, RC, isStackAligned, STI, false); 3282 } 3283 3284 3285 static unsigned getLoadRegOpcode(unsigned DestReg, 3286 const TargetRegisterClass *RC, 3287 bool isStackAligned, 3288 const X86Subtarget &STI) { 3289 return getLoadStoreRegOpcode(DestReg, RC, isStackAligned, STI, true); 3290 } 3291 3292 void X86InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB, 3293 MachineBasicBlock::iterator MI, 3294 unsigned SrcReg, bool isKill, int FrameIdx, 3295 const TargetRegisterClass *RC, 3296 const TargetRegisterInfo *TRI) const { 3297 const MachineFunction &MF = *MBB.getParent(); 3298 assert(MF.getFrameInfo().getObjectSize(FrameIdx) >= TRI->getSpillSize(*RC) && 3299 "Stack slot too small for store"); 3300 unsigned Alignment = std::max<uint32_t>(TRI->getSpillSize(*RC), 16); 3301 bool isAligned = 3302 (Subtarget.getFrameLowering()->getStackAlignment() >= Alignment) || 3303 RI.canRealignStack(MF); 3304 unsigned Opc = getStoreRegOpcode(SrcReg, RC, isAligned, Subtarget); 3305 DebugLoc DL = MBB.findDebugLoc(MI); 3306 addFrameReference(BuildMI(MBB, MI, DL, get(Opc)), FrameIdx) 3307 .addReg(SrcReg, getKillRegState(isKill)); 3308 } 3309 3310 void X86InstrInfo::storeRegToAddr(MachineFunction &MF, unsigned SrcReg, 3311 bool isKill, 3312 SmallVectorImpl<MachineOperand> &Addr, 3313 const TargetRegisterClass *RC, 3314 MachineInstr::mmo_iterator MMOBegin, 3315 MachineInstr::mmo_iterator MMOEnd, 3316 SmallVectorImpl<MachineInstr*> &NewMIs) const { 3317 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 3318 unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16); 3319 bool isAligned = MMOBegin != MMOEnd && 3320 (*MMOBegin)->getAlignment() >= Alignment; 3321 unsigned Opc = getStoreRegOpcode(SrcReg, RC, isAligned, Subtarget); 3322 DebugLoc DL; 3323 MachineInstrBuilder MIB = BuildMI(MF, DL, get(Opc)); 3324 for (unsigned i = 0, e = Addr.size(); i != e; ++i) 3325 MIB.add(Addr[i]); 3326 MIB.addReg(SrcReg, getKillRegState(isKill)); 3327 (*MIB).setMemRefs(MMOBegin, MMOEnd); 3328 NewMIs.push_back(MIB); 3329 } 3330 3331 3332 void X86InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB, 3333 MachineBasicBlock::iterator MI, 3334 unsigned DestReg, int FrameIdx, 3335 const TargetRegisterClass *RC, 3336 const TargetRegisterInfo *TRI) const { 3337 const MachineFunction &MF = *MBB.getParent(); 3338 unsigned Alignment = std::max<uint32_t>(TRI->getSpillSize(*RC), 16); 3339 bool isAligned = 3340 (Subtarget.getFrameLowering()->getStackAlignment() >= Alignment) || 3341 RI.canRealignStack(MF); 3342 unsigned Opc = getLoadRegOpcode(DestReg, RC, isAligned, Subtarget); 3343 DebugLoc DL = MBB.findDebugLoc(MI); 3344 addFrameReference(BuildMI(MBB, MI, DL, get(Opc), DestReg), FrameIdx); 3345 } 3346 3347 void X86InstrInfo::loadRegFromAddr(MachineFunction &MF, unsigned DestReg, 3348 SmallVectorImpl<MachineOperand> &Addr, 3349 const TargetRegisterClass *RC, 3350 MachineInstr::mmo_iterator MMOBegin, 3351 MachineInstr::mmo_iterator MMOEnd, 3352 SmallVectorImpl<MachineInstr*> &NewMIs) const { 3353 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 3354 unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16); 3355 bool isAligned = MMOBegin != MMOEnd && 3356 (*MMOBegin)->getAlignment() >= Alignment; 3357 unsigned Opc = getLoadRegOpcode(DestReg, RC, isAligned, Subtarget); 3358 DebugLoc DL; 3359 MachineInstrBuilder MIB = BuildMI(MF, DL, get(Opc), DestReg); 3360 for (unsigned i = 0, e = Addr.size(); i != e; ++i) 3361 MIB.add(Addr[i]); 3362 (*MIB).setMemRefs(MMOBegin, MMOEnd); 3363 NewMIs.push_back(MIB); 3364 } 3365 3366 bool X86InstrInfo::analyzeCompare(const MachineInstr &MI, unsigned &SrcReg, 3367 unsigned &SrcReg2, int &CmpMask, 3368 int &CmpValue) const { 3369 switch (MI.getOpcode()) { 3370 default: break; 3371 case X86::CMP64ri32: 3372 case X86::CMP64ri8: 3373 case X86::CMP32ri: 3374 case X86::CMP32ri8: 3375 case X86::CMP16ri: 3376 case X86::CMP16ri8: 3377 case X86::CMP8ri: 3378 SrcReg = MI.getOperand(0).getReg(); 3379 SrcReg2 = 0; 3380 if (MI.getOperand(1).isImm()) { 3381 CmpMask = ~0; 3382 CmpValue = MI.getOperand(1).getImm(); 3383 } else { 3384 CmpMask = CmpValue = 0; 3385 } 3386 return true; 3387 // A SUB can be used to perform comparison. 3388 case X86::SUB64rm: 3389 case X86::SUB32rm: 3390 case X86::SUB16rm: 3391 case X86::SUB8rm: 3392 SrcReg = MI.getOperand(1).getReg(); 3393 SrcReg2 = 0; 3394 CmpMask = 0; 3395 CmpValue = 0; 3396 return true; 3397 case X86::SUB64rr: 3398 case X86::SUB32rr: 3399 case X86::SUB16rr: 3400 case X86::SUB8rr: 3401 SrcReg = MI.getOperand(1).getReg(); 3402 SrcReg2 = MI.getOperand(2).getReg(); 3403 CmpMask = 0; 3404 CmpValue = 0; 3405 return true; 3406 case X86::SUB64ri32: 3407 case X86::SUB64ri8: 3408 case X86::SUB32ri: 3409 case X86::SUB32ri8: 3410 case X86::SUB16ri: 3411 case X86::SUB16ri8: 3412 case X86::SUB8ri: 3413 SrcReg = MI.getOperand(1).getReg(); 3414 SrcReg2 = 0; 3415 if (MI.getOperand(2).isImm()) { 3416 CmpMask = ~0; 3417 CmpValue = MI.getOperand(2).getImm(); 3418 } else { 3419 CmpMask = CmpValue = 0; 3420 } 3421 return true; 3422 case X86::CMP64rr: 3423 case X86::CMP32rr: 3424 case X86::CMP16rr: 3425 case X86::CMP8rr: 3426 SrcReg = MI.getOperand(0).getReg(); 3427 SrcReg2 = MI.getOperand(1).getReg(); 3428 CmpMask = 0; 3429 CmpValue = 0; 3430 return true; 3431 case X86::TEST8rr: 3432 case X86::TEST16rr: 3433 case X86::TEST32rr: 3434 case X86::TEST64rr: 3435 SrcReg = MI.getOperand(0).getReg(); 3436 if (MI.getOperand(1).getReg() != SrcReg) 3437 return false; 3438 // Compare against zero. 3439 SrcReg2 = 0; 3440 CmpMask = ~0; 3441 CmpValue = 0; 3442 return true; 3443 } 3444 return false; 3445 } 3446 3447 /// Check whether the first instruction, whose only 3448 /// purpose is to update flags, can be made redundant. 3449 /// CMPrr can be made redundant by SUBrr if the operands are the same. 3450 /// This function can be extended later on. 3451 /// SrcReg, SrcRegs: register operands for FlagI. 3452 /// ImmValue: immediate for FlagI if it takes an immediate. 3453 inline static bool isRedundantFlagInstr(MachineInstr &FlagI, unsigned SrcReg, 3454 unsigned SrcReg2, int ImmMask, 3455 int ImmValue, MachineInstr &OI) { 3456 if (((FlagI.getOpcode() == X86::CMP64rr && OI.getOpcode() == X86::SUB64rr) || 3457 (FlagI.getOpcode() == X86::CMP32rr && OI.getOpcode() == X86::SUB32rr) || 3458 (FlagI.getOpcode() == X86::CMP16rr && OI.getOpcode() == X86::SUB16rr) || 3459 (FlagI.getOpcode() == X86::CMP8rr && OI.getOpcode() == X86::SUB8rr)) && 3460 ((OI.getOperand(1).getReg() == SrcReg && 3461 OI.getOperand(2).getReg() == SrcReg2) || 3462 (OI.getOperand(1).getReg() == SrcReg2 && 3463 OI.getOperand(2).getReg() == SrcReg))) 3464 return true; 3465 3466 if (ImmMask != 0 && 3467 ((FlagI.getOpcode() == X86::CMP64ri32 && 3468 OI.getOpcode() == X86::SUB64ri32) || 3469 (FlagI.getOpcode() == X86::CMP64ri8 && 3470 OI.getOpcode() == X86::SUB64ri8) || 3471 (FlagI.getOpcode() == X86::CMP32ri && OI.getOpcode() == X86::SUB32ri) || 3472 (FlagI.getOpcode() == X86::CMP32ri8 && 3473 OI.getOpcode() == X86::SUB32ri8) || 3474 (FlagI.getOpcode() == X86::CMP16ri && OI.getOpcode() == X86::SUB16ri) || 3475 (FlagI.getOpcode() == X86::CMP16ri8 && 3476 OI.getOpcode() == X86::SUB16ri8) || 3477 (FlagI.getOpcode() == X86::CMP8ri && OI.getOpcode() == X86::SUB8ri)) && 3478 OI.getOperand(1).getReg() == SrcReg && 3479 OI.getOperand(2).getImm() == ImmValue) 3480 return true; 3481 return false; 3482 } 3483 3484 /// Check whether the definition can be converted 3485 /// to remove a comparison against zero. 3486 inline static bool isDefConvertible(MachineInstr &MI) { 3487 switch (MI.getOpcode()) { 3488 default: return false; 3489 3490 // The shift instructions only modify ZF if their shift count is non-zero. 3491 // N.B.: The processor truncates the shift count depending on the encoding. 3492 case X86::SAR8ri: case X86::SAR16ri: case X86::SAR32ri:case X86::SAR64ri: 3493 case X86::SHR8ri: case X86::SHR16ri: case X86::SHR32ri:case X86::SHR64ri: 3494 return getTruncatedShiftCount(MI, 2) != 0; 3495 3496 // Some left shift instructions can be turned into LEA instructions but only 3497 // if their flags aren't used. Avoid transforming such instructions. 3498 case X86::SHL8ri: case X86::SHL16ri: case X86::SHL32ri:case X86::SHL64ri:{ 3499 unsigned ShAmt = getTruncatedShiftCount(MI, 2); 3500 if (isTruncatedShiftCountForLEA(ShAmt)) return false; 3501 return ShAmt != 0; 3502 } 3503 3504 case X86::SHRD16rri8:case X86::SHRD32rri8:case X86::SHRD64rri8: 3505 case X86::SHLD16rri8:case X86::SHLD32rri8:case X86::SHLD64rri8: 3506 return getTruncatedShiftCount(MI, 3) != 0; 3507 3508 case X86::SUB64ri32: case X86::SUB64ri8: case X86::SUB32ri: 3509 case X86::SUB32ri8: case X86::SUB16ri: case X86::SUB16ri8: 3510 case X86::SUB8ri: case X86::SUB64rr: case X86::SUB32rr: 3511 case X86::SUB16rr: case X86::SUB8rr: case X86::SUB64rm: 3512 case X86::SUB32rm: case X86::SUB16rm: case X86::SUB8rm: 3513 case X86::DEC64r: case X86::DEC32r: case X86::DEC16r: case X86::DEC8r: 3514 case X86::ADD64ri32: case X86::ADD64ri8: case X86::ADD32ri: 3515 case X86::ADD32ri8: case X86::ADD16ri: case X86::ADD16ri8: 3516 case X86::ADD8ri: case X86::ADD64rr: case X86::ADD32rr: 3517 case X86::ADD16rr: case X86::ADD8rr: case X86::ADD64rm: 3518 case X86::ADD32rm: case X86::ADD16rm: case X86::ADD8rm: 3519 case X86::INC64r: case X86::INC32r: case X86::INC16r: case X86::INC8r: 3520 case X86::AND64ri32: case X86::AND64ri8: case X86::AND32ri: 3521 case X86::AND32ri8: case X86::AND16ri: case X86::AND16ri8: 3522 case X86::AND8ri: case X86::AND64rr: case X86::AND32rr: 3523 case X86::AND16rr: case X86::AND8rr: case X86::AND64rm: 3524 case X86::AND32rm: case X86::AND16rm: case X86::AND8rm: 3525 case X86::XOR64ri32: case X86::XOR64ri8: case X86::XOR32ri: 3526 case X86::XOR32ri8: case X86::XOR16ri: case X86::XOR16ri8: 3527 case X86::XOR8ri: case X86::XOR64rr: case X86::XOR32rr: 3528 case X86::XOR16rr: case X86::XOR8rr: case X86::XOR64rm: 3529 case X86::XOR32rm: case X86::XOR16rm: case X86::XOR8rm: 3530 case X86::OR64ri32: case X86::OR64ri8: case X86::OR32ri: 3531 case X86::OR32ri8: case X86::OR16ri: case X86::OR16ri8: 3532 case X86::OR8ri: case X86::OR64rr: case X86::OR32rr: 3533 case X86::OR16rr: case X86::OR8rr: case X86::OR64rm: 3534 case X86::OR32rm: case X86::OR16rm: case X86::OR8rm: 3535 case X86::ADC64ri32: case X86::ADC64ri8: case X86::ADC32ri: 3536 case X86::ADC32ri8: case X86::ADC16ri: case X86::ADC16ri8: 3537 case X86::ADC8ri: case X86::ADC64rr: case X86::ADC32rr: 3538 case X86::ADC16rr: case X86::ADC8rr: case X86::ADC64rm: 3539 case X86::ADC32rm: case X86::ADC16rm: case X86::ADC8rm: 3540 case X86::SBB64ri32: case X86::SBB64ri8: case X86::SBB32ri: 3541 case X86::SBB32ri8: case X86::SBB16ri: case X86::SBB16ri8: 3542 case X86::SBB8ri: case X86::SBB64rr: case X86::SBB32rr: 3543 case X86::SBB16rr: case X86::SBB8rr: case X86::SBB64rm: 3544 case X86::SBB32rm: case X86::SBB16rm: case X86::SBB8rm: 3545 case X86::NEG8r: case X86::NEG16r: case X86::NEG32r: case X86::NEG64r: 3546 case X86::SAR8r1: case X86::SAR16r1: case X86::SAR32r1:case X86::SAR64r1: 3547 case X86::SHR8r1: case X86::SHR16r1: case X86::SHR32r1:case X86::SHR64r1: 3548 case X86::SHL8r1: case X86::SHL16r1: case X86::SHL32r1:case X86::SHL64r1: 3549 case X86::ANDN32rr: case X86::ANDN32rm: 3550 case X86::ANDN64rr: case X86::ANDN64rm: 3551 case X86::BEXTR32rr: case X86::BEXTR64rr: 3552 case X86::BEXTR32rm: case X86::BEXTR64rm: 3553 case X86::BLSI32rr: case X86::BLSI32rm: 3554 case X86::BLSI64rr: case X86::BLSI64rm: 3555 case X86::BLSMSK32rr:case X86::BLSMSK32rm: 3556 case X86::BLSMSK64rr:case X86::BLSMSK64rm: 3557 case X86::BLSR32rr: case X86::BLSR32rm: 3558 case X86::BLSR64rr: case X86::BLSR64rm: 3559 case X86::BZHI32rr: case X86::BZHI32rm: 3560 case X86::BZHI64rr: case X86::BZHI64rm: 3561 case X86::LZCNT16rr: case X86::LZCNT16rm: 3562 case X86::LZCNT32rr: case X86::LZCNT32rm: 3563 case X86::LZCNT64rr: case X86::LZCNT64rm: 3564 case X86::POPCNT16rr:case X86::POPCNT16rm: 3565 case X86::POPCNT32rr:case X86::POPCNT32rm: 3566 case X86::POPCNT64rr:case X86::POPCNT64rm: 3567 case X86::TZCNT16rr: case X86::TZCNT16rm: 3568 case X86::TZCNT32rr: case X86::TZCNT32rm: 3569 case X86::TZCNT64rr: case X86::TZCNT64rm: 3570 case X86::BEXTRI32ri: case X86::BEXTRI32mi: 3571 case X86::BEXTRI64ri: case X86::BEXTRI64mi: 3572 case X86::BLCFILL32rr: case X86::BLCFILL32rm: 3573 case X86::BLCFILL64rr: case X86::BLCFILL64rm: 3574 case X86::BLCI32rr: case X86::BLCI32rm: 3575 case X86::BLCI64rr: case X86::BLCI64rm: 3576 case X86::BLCIC32rr: case X86::BLCIC32rm: 3577 case X86::BLCIC64rr: case X86::BLCIC64rm: 3578 case X86::BLCMSK32rr: case X86::BLCMSK32rm: 3579 case X86::BLCMSK64rr: case X86::BLCMSK64rm: 3580 case X86::BLCS32rr: case X86::BLCS32rm: 3581 case X86::BLCS64rr: case X86::BLCS64rm: 3582 case X86::BLSFILL32rr: case X86::BLSFILL32rm: 3583 case X86::BLSFILL64rr: case X86::BLSFILL64rm: 3584 case X86::BLSIC32rr: case X86::BLSIC32rm: 3585 case X86::BLSIC64rr: case X86::BLSIC64rm: 3586 return true; 3587 } 3588 } 3589 3590 /// Check whether the use can be converted to remove a comparison against zero. 3591 static X86::CondCode isUseDefConvertible(MachineInstr &MI) { 3592 switch (MI.getOpcode()) { 3593 default: return X86::COND_INVALID; 3594 case X86::LZCNT16rr: case X86::LZCNT16rm: 3595 case X86::LZCNT32rr: case X86::LZCNT32rm: 3596 case X86::LZCNT64rr: case X86::LZCNT64rm: 3597 return X86::COND_B; 3598 case X86::POPCNT16rr:case X86::POPCNT16rm: 3599 case X86::POPCNT32rr:case X86::POPCNT32rm: 3600 case X86::POPCNT64rr:case X86::POPCNT64rm: 3601 return X86::COND_E; 3602 case X86::TZCNT16rr: case X86::TZCNT16rm: 3603 case X86::TZCNT32rr: case X86::TZCNT32rm: 3604 case X86::TZCNT64rr: case X86::TZCNT64rm: 3605 return X86::COND_B; 3606 case X86::BSF16rr: 3607 case X86::BSF16rm: 3608 case X86::BSF32rr: 3609 case X86::BSF32rm: 3610 case X86::BSF64rr: 3611 case X86::BSF64rm: 3612 return X86::COND_E; 3613 } 3614 } 3615 3616 /// Check if there exists an earlier instruction that 3617 /// operates on the same source operands and sets flags in the same way as 3618 /// Compare; remove Compare if possible. 3619 bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, unsigned SrcReg, 3620 unsigned SrcReg2, int CmpMask, 3621 int CmpValue, 3622 const MachineRegisterInfo *MRI) const { 3623 // Check whether we can replace SUB with CMP. 3624 unsigned NewOpcode = 0; 3625 switch (CmpInstr.getOpcode()) { 3626 default: break; 3627 case X86::SUB64ri32: 3628 case X86::SUB64ri8: 3629 case X86::SUB32ri: 3630 case X86::SUB32ri8: 3631 case X86::SUB16ri: 3632 case X86::SUB16ri8: 3633 case X86::SUB8ri: 3634 case X86::SUB64rm: 3635 case X86::SUB32rm: 3636 case X86::SUB16rm: 3637 case X86::SUB8rm: 3638 case X86::SUB64rr: 3639 case X86::SUB32rr: 3640 case X86::SUB16rr: 3641 case X86::SUB8rr: { 3642 if (!MRI->use_nodbg_empty(CmpInstr.getOperand(0).getReg())) 3643 return false; 3644 // There is no use of the destination register, we can replace SUB with CMP. 3645 switch (CmpInstr.getOpcode()) { 3646 default: llvm_unreachable("Unreachable!"); 3647 case X86::SUB64rm: NewOpcode = X86::CMP64rm; break; 3648 case X86::SUB32rm: NewOpcode = X86::CMP32rm; break; 3649 case X86::SUB16rm: NewOpcode = X86::CMP16rm; break; 3650 case X86::SUB8rm: NewOpcode = X86::CMP8rm; break; 3651 case X86::SUB64rr: NewOpcode = X86::CMP64rr; break; 3652 case X86::SUB32rr: NewOpcode = X86::CMP32rr; break; 3653 case X86::SUB16rr: NewOpcode = X86::CMP16rr; break; 3654 case X86::SUB8rr: NewOpcode = X86::CMP8rr; break; 3655 case X86::SUB64ri32: NewOpcode = X86::CMP64ri32; break; 3656 case X86::SUB64ri8: NewOpcode = X86::CMP64ri8; break; 3657 case X86::SUB32ri: NewOpcode = X86::CMP32ri; break; 3658 case X86::SUB32ri8: NewOpcode = X86::CMP32ri8; break; 3659 case X86::SUB16ri: NewOpcode = X86::CMP16ri; break; 3660 case X86::SUB16ri8: NewOpcode = X86::CMP16ri8; break; 3661 case X86::SUB8ri: NewOpcode = X86::CMP8ri; break; 3662 } 3663 CmpInstr.setDesc(get(NewOpcode)); 3664 CmpInstr.RemoveOperand(0); 3665 // Fall through to optimize Cmp if Cmp is CMPrr or CMPri. 3666 if (NewOpcode == X86::CMP64rm || NewOpcode == X86::CMP32rm || 3667 NewOpcode == X86::CMP16rm || NewOpcode == X86::CMP8rm) 3668 return false; 3669 } 3670 } 3671 3672 // Get the unique definition of SrcReg. 3673 MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg); 3674 if (!MI) return false; 3675 3676 // CmpInstr is the first instruction of the BB. 3677 MachineBasicBlock::iterator I = CmpInstr, Def = MI; 3678 3679 // If we are comparing against zero, check whether we can use MI to update 3680 // EFLAGS. If MI is not in the same BB as CmpInstr, do not optimize. 3681 bool IsCmpZero = (CmpMask != 0 && CmpValue == 0); 3682 if (IsCmpZero && MI->getParent() != CmpInstr.getParent()) 3683 return false; 3684 3685 // If we have a use of the source register between the def and our compare 3686 // instruction we can eliminate the compare iff the use sets EFLAGS in the 3687 // right way. 3688 bool ShouldUpdateCC = false; 3689 X86::CondCode NewCC = X86::COND_INVALID; 3690 if (IsCmpZero && !isDefConvertible(*MI)) { 3691 // Scan forward from the use until we hit the use we're looking for or the 3692 // compare instruction. 3693 for (MachineBasicBlock::iterator J = MI;; ++J) { 3694 // Do we have a convertible instruction? 3695 NewCC = isUseDefConvertible(*J); 3696 if (NewCC != X86::COND_INVALID && J->getOperand(1).isReg() && 3697 J->getOperand(1).getReg() == SrcReg) { 3698 assert(J->definesRegister(X86::EFLAGS) && "Must be an EFLAGS def!"); 3699 ShouldUpdateCC = true; // Update CC later on. 3700 // This is not a def of SrcReg, but still a def of EFLAGS. Keep going 3701 // with the new def. 3702 Def = J; 3703 MI = &*Def; 3704 break; 3705 } 3706 3707 if (J == I) 3708 return false; 3709 } 3710 } 3711 3712 // We are searching for an earlier instruction that can make CmpInstr 3713 // redundant and that instruction will be saved in Sub. 3714 MachineInstr *Sub = nullptr; 3715 const TargetRegisterInfo *TRI = &getRegisterInfo(); 3716 3717 // We iterate backward, starting from the instruction before CmpInstr and 3718 // stop when reaching the definition of a source register or done with the BB. 3719 // RI points to the instruction before CmpInstr. 3720 // If the definition is in this basic block, RE points to the definition; 3721 // otherwise, RE is the rend of the basic block. 3722 MachineBasicBlock::reverse_iterator 3723 RI = ++I.getReverse(), 3724 RE = CmpInstr.getParent() == MI->getParent() 3725 ? Def.getReverse() /* points to MI */ 3726 : CmpInstr.getParent()->rend(); 3727 MachineInstr *Movr0Inst = nullptr; 3728 for (; RI != RE; ++RI) { 3729 MachineInstr &Instr = *RI; 3730 // Check whether CmpInstr can be made redundant by the current instruction. 3731 if (!IsCmpZero && isRedundantFlagInstr(CmpInstr, SrcReg, SrcReg2, CmpMask, 3732 CmpValue, Instr)) { 3733 Sub = &Instr; 3734 break; 3735 } 3736 3737 if (Instr.modifiesRegister(X86::EFLAGS, TRI) || 3738 Instr.readsRegister(X86::EFLAGS, TRI)) { 3739 // This instruction modifies or uses EFLAGS. 3740 3741 // MOV32r0 etc. are implemented with xor which clobbers condition code. 3742 // They are safe to move up, if the definition to EFLAGS is dead and 3743 // earlier instructions do not read or write EFLAGS. 3744 if (!Movr0Inst && Instr.getOpcode() == X86::MOV32r0 && 3745 Instr.registerDefIsDead(X86::EFLAGS, TRI)) { 3746 Movr0Inst = &Instr; 3747 continue; 3748 } 3749 3750 // We can't remove CmpInstr. 3751 return false; 3752 } 3753 } 3754 3755 // Return false if no candidates exist. 3756 if (!IsCmpZero && !Sub) 3757 return false; 3758 3759 bool IsSwapped = (SrcReg2 != 0 && Sub->getOperand(1).getReg() == SrcReg2 && 3760 Sub->getOperand(2).getReg() == SrcReg); 3761 3762 // Scan forward from the instruction after CmpInstr for uses of EFLAGS. 3763 // It is safe to remove CmpInstr if EFLAGS is redefined or killed. 3764 // If we are done with the basic block, we need to check whether EFLAGS is 3765 // live-out. 3766 bool IsSafe = false; 3767 SmallVector<std::pair<MachineInstr*, unsigned /*NewOpc*/>, 4> OpsToUpdate; 3768 MachineBasicBlock::iterator E = CmpInstr.getParent()->end(); 3769 for (++I; I != E; ++I) { 3770 const MachineInstr &Instr = *I; 3771 bool ModifyEFLAGS = Instr.modifiesRegister(X86::EFLAGS, TRI); 3772 bool UseEFLAGS = Instr.readsRegister(X86::EFLAGS, TRI); 3773 // We should check the usage if this instruction uses and updates EFLAGS. 3774 if (!UseEFLAGS && ModifyEFLAGS) { 3775 // It is safe to remove CmpInstr if EFLAGS is updated again. 3776 IsSafe = true; 3777 break; 3778 } 3779 if (!UseEFLAGS && !ModifyEFLAGS) 3780 continue; 3781 3782 // EFLAGS is used by this instruction. 3783 X86::CondCode OldCC = X86::COND_INVALID; 3784 bool OpcIsSET = false; 3785 if (IsCmpZero || IsSwapped) { 3786 // We decode the condition code from opcode. 3787 if (Instr.isBranch()) 3788 OldCC = X86::getCondFromBranchOpc(Instr.getOpcode()); 3789 else { 3790 OldCC = X86::getCondFromSETOpc(Instr.getOpcode()); 3791 if (OldCC != X86::COND_INVALID) 3792 OpcIsSET = true; 3793 else 3794 OldCC = X86::getCondFromCMovOpc(Instr.getOpcode()); 3795 } 3796 if (OldCC == X86::COND_INVALID) return false; 3797 } 3798 X86::CondCode ReplacementCC = X86::COND_INVALID; 3799 if (IsCmpZero) { 3800 switch (OldCC) { 3801 default: break; 3802 case X86::COND_A: case X86::COND_AE: 3803 case X86::COND_B: case X86::COND_BE: 3804 case X86::COND_G: case X86::COND_GE: 3805 case X86::COND_L: case X86::COND_LE: 3806 case X86::COND_O: case X86::COND_NO: 3807 // CF and OF are used, we can't perform this optimization. 3808 return false; 3809 } 3810 3811 // If we're updating the condition code check if we have to reverse the 3812 // condition. 3813 if (ShouldUpdateCC) 3814 switch (OldCC) { 3815 default: 3816 return false; 3817 case X86::COND_E: 3818 ReplacementCC = NewCC; 3819 break; 3820 case X86::COND_NE: 3821 ReplacementCC = GetOppositeBranchCondition(NewCC); 3822 break; 3823 } 3824 } else if (IsSwapped) { 3825 // If we have SUB(r1, r2) and CMP(r2, r1), the condition code needs 3826 // to be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc. 3827 // We swap the condition code and synthesize the new opcode. 3828 ReplacementCC = getSwappedCondition(OldCC); 3829 if (ReplacementCC == X86::COND_INVALID) return false; 3830 } 3831 3832 if ((ShouldUpdateCC || IsSwapped) && ReplacementCC != OldCC) { 3833 // Synthesize the new opcode. 3834 bool HasMemoryOperand = Instr.hasOneMemOperand(); 3835 unsigned NewOpc; 3836 if (Instr.isBranch()) 3837 NewOpc = GetCondBranchFromCond(ReplacementCC); 3838 else if(OpcIsSET) 3839 NewOpc = getSETFromCond(ReplacementCC, HasMemoryOperand); 3840 else { 3841 unsigned DstReg = Instr.getOperand(0).getReg(); 3842 const TargetRegisterClass *DstRC = MRI->getRegClass(DstReg); 3843 NewOpc = getCMovFromCond(ReplacementCC, TRI->getRegSizeInBits(*DstRC)/8, 3844 HasMemoryOperand); 3845 } 3846 3847 // Push the MachineInstr to OpsToUpdate. 3848 // If it is safe to remove CmpInstr, the condition code of these 3849 // instructions will be modified. 3850 OpsToUpdate.push_back(std::make_pair(&*I, NewOpc)); 3851 } 3852 if (ModifyEFLAGS || Instr.killsRegister(X86::EFLAGS, TRI)) { 3853 // It is safe to remove CmpInstr if EFLAGS is updated again or killed. 3854 IsSafe = true; 3855 break; 3856 } 3857 } 3858 3859 // If EFLAGS is not killed nor re-defined, we should check whether it is 3860 // live-out. If it is live-out, do not optimize. 3861 if ((IsCmpZero || IsSwapped) && !IsSafe) { 3862 MachineBasicBlock *MBB = CmpInstr.getParent(); 3863 for (MachineBasicBlock *Successor : MBB->successors()) 3864 if (Successor->isLiveIn(X86::EFLAGS)) 3865 return false; 3866 } 3867 3868 // The instruction to be updated is either Sub or MI. 3869 Sub = IsCmpZero ? MI : Sub; 3870 // Move Movr0Inst to the appropriate place before Sub. 3871 if (Movr0Inst) { 3872 // Look backwards until we find a def that doesn't use the current EFLAGS. 3873 Def = Sub; 3874 MachineBasicBlock::reverse_iterator InsertI = Def.getReverse(), 3875 InsertE = Sub->getParent()->rend(); 3876 for (; InsertI != InsertE; ++InsertI) { 3877 MachineInstr *Instr = &*InsertI; 3878 if (!Instr->readsRegister(X86::EFLAGS, TRI) && 3879 Instr->modifiesRegister(X86::EFLAGS, TRI)) { 3880 Sub->getParent()->remove(Movr0Inst); 3881 Instr->getParent()->insert(MachineBasicBlock::iterator(Instr), 3882 Movr0Inst); 3883 break; 3884 } 3885 } 3886 if (InsertI == InsertE) 3887 return false; 3888 } 3889 3890 // Make sure Sub instruction defines EFLAGS and mark the def live. 3891 unsigned i = 0, e = Sub->getNumOperands(); 3892 for (; i != e; ++i) { 3893 MachineOperand &MO = Sub->getOperand(i); 3894 if (MO.isReg() && MO.isDef() && MO.getReg() == X86::EFLAGS) { 3895 MO.setIsDead(false); 3896 break; 3897 } 3898 } 3899 assert(i != e && "Unable to locate a def EFLAGS operand"); 3900 3901 CmpInstr.eraseFromParent(); 3902 3903 // Modify the condition code of instructions in OpsToUpdate. 3904 for (auto &Op : OpsToUpdate) 3905 Op.first->setDesc(get(Op.second)); 3906 return true; 3907 } 3908 3909 /// Try to remove the load by folding it to a register 3910 /// operand at the use. We fold the load instructions if load defines a virtual 3911 /// register, the virtual register is used once in the same BB, and the 3912 /// instructions in-between do not load or store, and have no side effects. 3913 MachineInstr *X86InstrInfo::optimizeLoadInstr(MachineInstr &MI, 3914 const MachineRegisterInfo *MRI, 3915 unsigned &FoldAsLoadDefReg, 3916 MachineInstr *&DefMI) const { 3917 // Check whether we can move DefMI here. 3918 DefMI = MRI->getVRegDef(FoldAsLoadDefReg); 3919 assert(DefMI); 3920 bool SawStore = false; 3921 if (!DefMI->isSafeToMove(nullptr, SawStore)) 3922 return nullptr; 3923 3924 // Collect information about virtual register operands of MI. 3925 SmallVector<unsigned, 1> SrcOperandIds; 3926 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 3927 MachineOperand &MO = MI.getOperand(i); 3928 if (!MO.isReg()) 3929 continue; 3930 unsigned Reg = MO.getReg(); 3931 if (Reg != FoldAsLoadDefReg) 3932 continue; 3933 // Do not fold if we have a subreg use or a def. 3934 if (MO.getSubReg() || MO.isDef()) 3935 return nullptr; 3936 SrcOperandIds.push_back(i); 3937 } 3938 if (SrcOperandIds.empty()) 3939 return nullptr; 3940 3941 // Check whether we can fold the def into SrcOperandId. 3942 if (MachineInstr *FoldMI = foldMemoryOperand(MI, SrcOperandIds, *DefMI)) { 3943 FoldAsLoadDefReg = 0; 3944 return FoldMI; 3945 } 3946 3947 return nullptr; 3948 } 3949 3950 /// Expand a single-def pseudo instruction to a two-addr 3951 /// instruction with two undef reads of the register being defined. 3952 /// This is used for mapping: 3953 /// %xmm4 = V_SET0 3954 /// to: 3955 /// %xmm4 = PXORrr undef %xmm4, undef %xmm4 3956 /// 3957 static bool Expand2AddrUndef(MachineInstrBuilder &MIB, 3958 const MCInstrDesc &Desc) { 3959 assert(Desc.getNumOperands() == 3 && "Expected two-addr instruction."); 3960 unsigned Reg = MIB->getOperand(0).getReg(); 3961 MIB->setDesc(Desc); 3962 3963 // MachineInstr::addOperand() will insert explicit operands before any 3964 // implicit operands. 3965 MIB.addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef); 3966 // But we don't trust that. 3967 assert(MIB->getOperand(1).getReg() == Reg && 3968 MIB->getOperand(2).getReg() == Reg && "Misplaced operand"); 3969 return true; 3970 } 3971 3972 /// Expand a single-def pseudo instruction to a two-addr 3973 /// instruction with two %k0 reads. 3974 /// This is used for mapping: 3975 /// %k4 = K_SET1 3976 /// to: 3977 /// %k4 = KXNORrr %k0, %k0 3978 static bool Expand2AddrKreg(MachineInstrBuilder &MIB, 3979 const MCInstrDesc &Desc, unsigned Reg) { 3980 assert(Desc.getNumOperands() == 3 && "Expected two-addr instruction."); 3981 MIB->setDesc(Desc); 3982 MIB.addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef); 3983 return true; 3984 } 3985 3986 static bool expandMOV32r1(MachineInstrBuilder &MIB, const TargetInstrInfo &TII, 3987 bool MinusOne) { 3988 MachineBasicBlock &MBB = *MIB->getParent(); 3989 DebugLoc DL = MIB->getDebugLoc(); 3990 unsigned Reg = MIB->getOperand(0).getReg(); 3991 3992 // Insert the XOR. 3993 BuildMI(MBB, MIB.getInstr(), DL, TII.get(X86::XOR32rr), Reg) 3994 .addReg(Reg, RegState::Undef) 3995 .addReg(Reg, RegState::Undef); 3996 3997 // Turn the pseudo into an INC or DEC. 3998 MIB->setDesc(TII.get(MinusOne ? X86::DEC32r : X86::INC32r)); 3999 MIB.addReg(Reg); 4000 4001 return true; 4002 } 4003 4004 static bool ExpandMOVImmSExti8(MachineInstrBuilder &MIB, 4005 const TargetInstrInfo &TII, 4006 const X86Subtarget &Subtarget) { 4007 MachineBasicBlock &MBB = *MIB->getParent(); 4008 DebugLoc DL = MIB->getDebugLoc(); 4009 int64_t Imm = MIB->getOperand(1).getImm(); 4010 assert(Imm != 0 && "Using push/pop for 0 is not efficient."); 4011 MachineBasicBlock::iterator I = MIB.getInstr(); 4012 4013 int StackAdjustment; 4014 4015 if (Subtarget.is64Bit()) { 4016 assert(MIB->getOpcode() == X86::MOV64ImmSExti8 || 4017 MIB->getOpcode() == X86::MOV32ImmSExti8); 4018 4019 // Can't use push/pop lowering if the function might write to the red zone. 4020 X86MachineFunctionInfo *X86FI = 4021 MBB.getParent()->getInfo<X86MachineFunctionInfo>(); 4022 if (X86FI->getUsesRedZone()) { 4023 MIB->setDesc(TII.get(MIB->getOpcode() == 4024 X86::MOV32ImmSExti8 ? X86::MOV32ri : X86::MOV64ri)); 4025 return true; 4026 } 4027 4028 // 64-bit mode doesn't have 32-bit push/pop, so use 64-bit operations and 4029 // widen the register if necessary. 4030 StackAdjustment = 8; 4031 BuildMI(MBB, I, DL, TII.get(X86::PUSH64i8)).addImm(Imm); 4032 MIB->setDesc(TII.get(X86::POP64r)); 4033 MIB->getOperand(0) 4034 .setReg(getX86SubSuperRegister(MIB->getOperand(0).getReg(), 64)); 4035 } else { 4036 assert(MIB->getOpcode() == X86::MOV32ImmSExti8); 4037 StackAdjustment = 4; 4038 BuildMI(MBB, I, DL, TII.get(X86::PUSH32i8)).addImm(Imm); 4039 MIB->setDesc(TII.get(X86::POP32r)); 4040 } 4041 4042 // Build CFI if necessary. 4043 MachineFunction &MF = *MBB.getParent(); 4044 const X86FrameLowering *TFL = Subtarget.getFrameLowering(); 4045 bool IsWin64Prologue = MF.getTarget().getMCAsmInfo()->usesWindowsCFI(); 4046 bool NeedsDwarfCFI = 4047 !IsWin64Prologue && 4048 (MF.getMMI().hasDebugInfo() || MF.getFunction().needsUnwindTableEntry()); 4049 bool EmitCFI = !TFL->hasFP(MF) && NeedsDwarfCFI; 4050 if (EmitCFI) { 4051 TFL->BuildCFI(MBB, I, DL, 4052 MCCFIInstruction::createAdjustCfaOffset(nullptr, StackAdjustment)); 4053 TFL->BuildCFI(MBB, std::next(I), DL, 4054 MCCFIInstruction::createAdjustCfaOffset(nullptr, -StackAdjustment)); 4055 } 4056 4057 return true; 4058 } 4059 4060 // LoadStackGuard has so far only been implemented for 64-bit MachO. Different 4061 // code sequence is needed for other targets. 4062 static void expandLoadStackGuard(MachineInstrBuilder &MIB, 4063 const TargetInstrInfo &TII) { 4064 MachineBasicBlock &MBB = *MIB->getParent(); 4065 DebugLoc DL = MIB->getDebugLoc(); 4066 unsigned Reg = MIB->getOperand(0).getReg(); 4067 const GlobalValue *GV = 4068 cast<GlobalValue>((*MIB->memoperands_begin())->getValue()); 4069 auto Flags = MachineMemOperand::MOLoad | 4070 MachineMemOperand::MODereferenceable | 4071 MachineMemOperand::MOInvariant; 4072 MachineMemOperand *MMO = MBB.getParent()->getMachineMemOperand( 4073 MachinePointerInfo::getGOT(*MBB.getParent()), Flags, 8, 8); 4074 MachineBasicBlock::iterator I = MIB.getInstr(); 4075 4076 BuildMI(MBB, I, DL, TII.get(X86::MOV64rm), Reg).addReg(X86::RIP).addImm(1) 4077 .addReg(0).addGlobalAddress(GV, 0, X86II::MO_GOTPCREL).addReg(0) 4078 .addMemOperand(MMO); 4079 MIB->setDebugLoc(DL); 4080 MIB->setDesc(TII.get(X86::MOV64rm)); 4081 MIB.addReg(Reg, RegState::Kill).addImm(1).addReg(0).addImm(0).addReg(0); 4082 } 4083 4084 static bool expandXorFP(MachineInstrBuilder &MIB, const TargetInstrInfo &TII) { 4085 MachineBasicBlock &MBB = *MIB->getParent(); 4086 MachineFunction &MF = *MBB.getParent(); 4087 const X86Subtarget &Subtarget = MF.getSubtarget<X86Subtarget>(); 4088 const X86RegisterInfo *TRI = Subtarget.getRegisterInfo(); 4089 unsigned XorOp = 4090 MIB->getOpcode() == X86::XOR64_FP ? X86::XOR64rr : X86::XOR32rr; 4091 MIB->setDesc(TII.get(XorOp)); 4092 MIB.addReg(TRI->getFrameRegister(MF), RegState::Undef); 4093 return true; 4094 } 4095 4096 // This is used to handle spills for 128/256-bit registers when we have AVX512, 4097 // but not VLX. If it uses an extended register we need to use an instruction 4098 // that loads the lower 128/256-bit, but is available with only AVX512F. 4099 static bool expandNOVLXLoad(MachineInstrBuilder &MIB, 4100 const TargetRegisterInfo *TRI, 4101 const MCInstrDesc &LoadDesc, 4102 const MCInstrDesc &BroadcastDesc, 4103 unsigned SubIdx) { 4104 unsigned DestReg = MIB->getOperand(0).getReg(); 4105 // Check if DestReg is XMM16-31 or YMM16-31. 4106 if (TRI->getEncodingValue(DestReg) < 16) { 4107 // We can use a normal VEX encoded load. 4108 MIB->setDesc(LoadDesc); 4109 } else { 4110 // Use a 128/256-bit VBROADCAST instruction. 4111 MIB->setDesc(BroadcastDesc); 4112 // Change the destination to a 512-bit register. 4113 DestReg = TRI->getMatchingSuperReg(DestReg, SubIdx, &X86::VR512RegClass); 4114 MIB->getOperand(0).setReg(DestReg); 4115 } 4116 return true; 4117 } 4118 4119 // This is used to handle spills for 128/256-bit registers when we have AVX512, 4120 // but not VLX. If it uses an extended register we need to use an instruction 4121 // that stores the lower 128/256-bit, but is available with only AVX512F. 4122 static bool expandNOVLXStore(MachineInstrBuilder &MIB, 4123 const TargetRegisterInfo *TRI, 4124 const MCInstrDesc &StoreDesc, 4125 const MCInstrDesc &ExtractDesc, 4126 unsigned SubIdx) { 4127 unsigned SrcReg = MIB->getOperand(X86::AddrNumOperands).getReg(); 4128 // Check if DestReg is XMM16-31 or YMM16-31. 4129 if (TRI->getEncodingValue(SrcReg) < 16) { 4130 // We can use a normal VEX encoded store. 4131 MIB->setDesc(StoreDesc); 4132 } else { 4133 // Use a VEXTRACTF instruction. 4134 MIB->setDesc(ExtractDesc); 4135 // Change the destination to a 512-bit register. 4136 SrcReg = TRI->getMatchingSuperReg(SrcReg, SubIdx, &X86::VR512RegClass); 4137 MIB->getOperand(X86::AddrNumOperands).setReg(SrcReg); 4138 MIB.addImm(0x0); // Append immediate to extract from the lower bits. 4139 } 4140 4141 return true; 4142 } 4143 bool X86InstrInfo::expandPostRAPseudo(MachineInstr &MI) const { 4144 bool HasAVX = Subtarget.hasAVX(); 4145 MachineInstrBuilder MIB(*MI.getParent()->getParent(), MI); 4146 switch (MI.getOpcode()) { 4147 case X86::MOV32r0: 4148 return Expand2AddrUndef(MIB, get(X86::XOR32rr)); 4149 case X86::MOV32r1: 4150 return expandMOV32r1(MIB, *this, /*MinusOne=*/ false); 4151 case X86::MOV32r_1: 4152 return expandMOV32r1(MIB, *this, /*MinusOne=*/ true); 4153 case X86::MOV32ImmSExti8: 4154 case X86::MOV64ImmSExti8: 4155 return ExpandMOVImmSExti8(MIB, *this, Subtarget); 4156 case X86::SETB_C8r: 4157 return Expand2AddrUndef(MIB, get(X86::SBB8rr)); 4158 case X86::SETB_C16r: 4159 return Expand2AddrUndef(MIB, get(X86::SBB16rr)); 4160 case X86::SETB_C32r: 4161 return Expand2AddrUndef(MIB, get(X86::SBB32rr)); 4162 case X86::SETB_C64r: 4163 return Expand2AddrUndef(MIB, get(X86::SBB64rr)); 4164 case X86::MMX_SET0: 4165 return Expand2AddrUndef(MIB, get(X86::MMX_PXORirr)); 4166 case X86::V_SET0: 4167 case X86::FsFLD0SS: 4168 case X86::FsFLD0SD: 4169 return Expand2AddrUndef(MIB, get(HasAVX ? X86::VXORPSrr : X86::XORPSrr)); 4170 case X86::AVX_SET0: { 4171 assert(HasAVX && "AVX not supported"); 4172 const TargetRegisterInfo *TRI = &getRegisterInfo(); 4173 unsigned SrcReg = MIB->getOperand(0).getReg(); 4174 unsigned XReg = TRI->getSubReg(SrcReg, X86::sub_xmm); 4175 MIB->getOperand(0).setReg(XReg); 4176 Expand2AddrUndef(MIB, get(X86::VXORPSrr)); 4177 MIB.addReg(SrcReg, RegState::ImplicitDefine); 4178 return true; 4179 } 4180 case X86::AVX512_128_SET0: 4181 case X86::AVX512_FsFLD0SS: 4182 case X86::AVX512_FsFLD0SD: { 4183 bool HasVLX = Subtarget.hasVLX(); 4184 unsigned SrcReg = MIB->getOperand(0).getReg(); 4185 const TargetRegisterInfo *TRI = &getRegisterInfo(); 4186 if (HasVLX || TRI->getEncodingValue(SrcReg) < 16) 4187 return Expand2AddrUndef(MIB, 4188 get(HasVLX ? X86::VPXORDZ128rr : X86::VXORPSrr)); 4189 // Extended register without VLX. Use a larger XOR. 4190 SrcReg = 4191 TRI->getMatchingSuperReg(SrcReg, X86::sub_xmm, &X86::VR512RegClass); 4192 MIB->getOperand(0).setReg(SrcReg); 4193 return Expand2AddrUndef(MIB, get(X86::VPXORDZrr)); 4194 } 4195 case X86::AVX512_256_SET0: 4196 case X86::AVX512_512_SET0: { 4197 bool HasVLX = Subtarget.hasVLX(); 4198 unsigned SrcReg = MIB->getOperand(0).getReg(); 4199 const TargetRegisterInfo *TRI = &getRegisterInfo(); 4200 if (HasVLX || TRI->getEncodingValue(SrcReg) < 16) { 4201 unsigned XReg = TRI->getSubReg(SrcReg, X86::sub_xmm); 4202 MIB->getOperand(0).setReg(XReg); 4203 Expand2AddrUndef(MIB, 4204 get(HasVLX ? X86::VPXORDZ128rr : X86::VXORPSrr)); 4205 MIB.addReg(SrcReg, RegState::ImplicitDefine); 4206 return true; 4207 } 4208 return Expand2AddrUndef(MIB, get(X86::VPXORDZrr)); 4209 } 4210 case X86::V_SETALLONES: 4211 return Expand2AddrUndef(MIB, get(HasAVX ? X86::VPCMPEQDrr : X86::PCMPEQDrr)); 4212 case X86::AVX2_SETALLONES: 4213 return Expand2AddrUndef(MIB, get(X86::VPCMPEQDYrr)); 4214 case X86::AVX1_SETALLONES: { 4215 unsigned Reg = MIB->getOperand(0).getReg(); 4216 // VCMPPSYrri with an immediate 0xf should produce VCMPTRUEPS. 4217 MIB->setDesc(get(X86::VCMPPSYrri)); 4218 MIB.addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef).addImm(0xf); 4219 return true; 4220 } 4221 case X86::AVX512_512_SETALLONES: { 4222 unsigned Reg = MIB->getOperand(0).getReg(); 4223 MIB->setDesc(get(X86::VPTERNLOGDZrri)); 4224 // VPTERNLOGD needs 3 register inputs and an immediate. 4225 // 0xff will return 1s for any input. 4226 MIB.addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef) 4227 .addReg(Reg, RegState::Undef).addImm(0xff); 4228 return true; 4229 } 4230 case X86::AVX512_512_SEXT_MASK_32: 4231 case X86::AVX512_512_SEXT_MASK_64: { 4232 unsigned Reg = MIB->getOperand(0).getReg(); 4233 unsigned MaskReg = MIB->getOperand(1).getReg(); 4234 unsigned MaskState = getRegState(MIB->getOperand(1)); 4235 unsigned Opc = (MI.getOpcode() == X86::AVX512_512_SEXT_MASK_64) ? 4236 X86::VPTERNLOGQZrrikz : X86::VPTERNLOGDZrrikz; 4237 MI.RemoveOperand(1); 4238 MIB->setDesc(get(Opc)); 4239 // VPTERNLOG needs 3 register inputs and an immediate. 4240 // 0xff will return 1s for any input. 4241 MIB.addReg(Reg, RegState::Undef).addReg(MaskReg, MaskState) 4242 .addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef).addImm(0xff); 4243 return true; 4244 } 4245 case X86::VMOVAPSZ128rm_NOVLX: 4246 return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVAPSrm), 4247 get(X86::VBROADCASTF32X4rm), X86::sub_xmm); 4248 case X86::VMOVUPSZ128rm_NOVLX: 4249 return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVUPSrm), 4250 get(X86::VBROADCASTF32X4rm), X86::sub_xmm); 4251 case X86::VMOVAPSZ256rm_NOVLX: 4252 return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVAPSYrm), 4253 get(X86::VBROADCASTF64X4rm), X86::sub_ymm); 4254 case X86::VMOVUPSZ256rm_NOVLX: 4255 return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVUPSYrm), 4256 get(X86::VBROADCASTF64X4rm), X86::sub_ymm); 4257 case X86::VMOVAPSZ128mr_NOVLX: 4258 return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVAPSmr), 4259 get(X86::VEXTRACTF32x4Zmr), X86::sub_xmm); 4260 case X86::VMOVUPSZ128mr_NOVLX: 4261 return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVUPSmr), 4262 get(X86::VEXTRACTF32x4Zmr), X86::sub_xmm); 4263 case X86::VMOVAPSZ256mr_NOVLX: 4264 return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVAPSYmr), 4265 get(X86::VEXTRACTF64x4Zmr), X86::sub_ymm); 4266 case X86::VMOVUPSZ256mr_NOVLX: 4267 return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVUPSYmr), 4268 get(X86::VEXTRACTF64x4Zmr), X86::sub_ymm); 4269 case X86::MOV32ri64: 4270 MI.setDesc(get(X86::MOV32ri)); 4271 return true; 4272 4273 // KNL does not recognize dependency-breaking idioms for mask registers, 4274 // so kxnor %k1, %k1, %k2 has a RAW dependence on %k1. 4275 // Using %k0 as the undef input register is a performance heuristic based 4276 // on the assumption that %k0 is used less frequently than the other mask 4277 // registers, since it is not usable as a write mask. 4278 // FIXME: A more advanced approach would be to choose the best input mask 4279 // register based on context. 4280 case X86::KSET0W: return Expand2AddrKreg(MIB, get(X86::KXORWrr), X86::K0); 4281 case X86::KSET0D: return Expand2AddrKreg(MIB, get(X86::KXORDrr), X86::K0); 4282 case X86::KSET0Q: return Expand2AddrKreg(MIB, get(X86::KXORQrr), X86::K0); 4283 case X86::KSET1W: return Expand2AddrKreg(MIB, get(X86::KXNORWrr), X86::K0); 4284 case X86::KSET1D: return Expand2AddrKreg(MIB, get(X86::KXNORDrr), X86::K0); 4285 case X86::KSET1Q: return Expand2AddrKreg(MIB, get(X86::KXNORQrr), X86::K0); 4286 case TargetOpcode::LOAD_STACK_GUARD: 4287 expandLoadStackGuard(MIB, *this); 4288 return true; 4289 case X86::XOR64_FP: 4290 case X86::XOR32_FP: 4291 return expandXorFP(MIB, *this); 4292 } 4293 return false; 4294 } 4295 4296 /// Return true for all instructions that only update 4297 /// the first 32 or 64-bits of the destination register and leave the rest 4298 /// unmodified. This can be used to avoid folding loads if the instructions 4299 /// only update part of the destination register, and the non-updated part is 4300 /// not needed. e.g. cvtss2sd, sqrtss. Unfolding the load from these 4301 /// instructions breaks the partial register dependency and it can improve 4302 /// performance. e.g.: 4303 /// 4304 /// movss (%rdi), %xmm0 4305 /// cvtss2sd %xmm0, %xmm0 4306 /// 4307 /// Instead of 4308 /// cvtss2sd (%rdi), %xmm0 4309 /// 4310 /// FIXME: This should be turned into a TSFlags. 4311 /// 4312 static bool hasPartialRegUpdate(unsigned Opcode, 4313 const X86Subtarget &Subtarget) { 4314 switch (Opcode) { 4315 case X86::CVTSI2SSrr: 4316 case X86::CVTSI2SSrm: 4317 case X86::CVTSI642SSrr: 4318 case X86::CVTSI642SSrm: 4319 case X86::CVTSI2SDrr: 4320 case X86::CVTSI2SDrm: 4321 case X86::CVTSI642SDrr: 4322 case X86::CVTSI642SDrm: 4323 case X86::CVTSD2SSrr: 4324 case X86::CVTSD2SSrm: 4325 case X86::CVTSS2SDrr: 4326 case X86::CVTSS2SDrm: 4327 case X86::MOVHPDrm: 4328 case X86::MOVHPSrm: 4329 case X86::MOVLPDrm: 4330 case X86::MOVLPSrm: 4331 case X86::RCPSSr: 4332 case X86::RCPSSm: 4333 case X86::RCPSSr_Int: 4334 case X86::RCPSSm_Int: 4335 case X86::ROUNDSDr: 4336 case X86::ROUNDSDm: 4337 case X86::ROUNDSSr: 4338 case X86::ROUNDSSm: 4339 case X86::RSQRTSSr: 4340 case X86::RSQRTSSm: 4341 case X86::RSQRTSSr_Int: 4342 case X86::RSQRTSSm_Int: 4343 case X86::SQRTSSr: 4344 case X86::SQRTSSm: 4345 case X86::SQRTSSr_Int: 4346 case X86::SQRTSSm_Int: 4347 case X86::SQRTSDr: 4348 case X86::SQRTSDm: 4349 case X86::SQRTSDr_Int: 4350 case X86::SQRTSDm_Int: 4351 return true; 4352 // GPR 4353 case X86::POPCNT32rm: 4354 case X86::POPCNT32rr: 4355 case X86::POPCNT64rm: 4356 case X86::POPCNT64rr: 4357 return Subtarget.hasPOPCNTFalseDeps(); 4358 case X86::LZCNT32rm: 4359 case X86::LZCNT32rr: 4360 case X86::LZCNT64rm: 4361 case X86::LZCNT64rr: 4362 case X86::TZCNT32rm: 4363 case X86::TZCNT32rr: 4364 case X86::TZCNT64rm: 4365 case X86::TZCNT64rr: 4366 return Subtarget.hasLZCNTFalseDeps(); 4367 } 4368 4369 return false; 4370 } 4371 4372 /// Inform the BreakFalseDeps pass how many idle 4373 /// instructions we would like before a partial register update. 4374 unsigned X86InstrInfo::getPartialRegUpdateClearance( 4375 const MachineInstr &MI, unsigned OpNum, 4376 const TargetRegisterInfo *TRI) const { 4377 if (OpNum != 0 || !hasPartialRegUpdate(MI.getOpcode(), Subtarget)) 4378 return 0; 4379 4380 // If MI is marked as reading Reg, the partial register update is wanted. 4381 const MachineOperand &MO = MI.getOperand(0); 4382 unsigned Reg = MO.getReg(); 4383 if (TargetRegisterInfo::isVirtualRegister(Reg)) { 4384 if (MO.readsReg() || MI.readsVirtualRegister(Reg)) 4385 return 0; 4386 } else { 4387 if (MI.readsRegister(Reg, TRI)) 4388 return 0; 4389 } 4390 4391 // If any instructions in the clearance range are reading Reg, insert a 4392 // dependency breaking instruction, which is inexpensive and is likely to 4393 // be hidden in other instruction's cycles. 4394 return PartialRegUpdateClearance; 4395 } 4396 4397 // Return true for any instruction the copies the high bits of the first source 4398 // operand into the unused high bits of the destination operand. 4399 static bool hasUndefRegUpdate(unsigned Opcode) { 4400 switch (Opcode) { 4401 case X86::VCVTSI2SSrr: 4402 case X86::VCVTSI2SSrm: 4403 case X86::VCVTSI2SSrr_Int: 4404 case X86::VCVTSI2SSrm_Int: 4405 case X86::VCVTSI642SSrr: 4406 case X86::VCVTSI642SSrm: 4407 case X86::VCVTSI642SSrr_Int: 4408 case X86::VCVTSI642SSrm_Int: 4409 case X86::VCVTSI2SDrr: 4410 case X86::VCVTSI2SDrm: 4411 case X86::VCVTSI2SDrr_Int: 4412 case X86::VCVTSI2SDrm_Int: 4413 case X86::VCVTSI642SDrr: 4414 case X86::VCVTSI642SDrm: 4415 case X86::VCVTSI642SDrr_Int: 4416 case X86::VCVTSI642SDrm_Int: 4417 case X86::VCVTSD2SSrr: 4418 case X86::VCVTSD2SSrm: 4419 case X86::VCVTSD2SSrr_Int: 4420 case X86::VCVTSD2SSrm_Int: 4421 case X86::VCVTSS2SDrr: 4422 case X86::VCVTSS2SDrm: 4423 case X86::VCVTSS2SDrr_Int: 4424 case X86::VCVTSS2SDrm_Int: 4425 case X86::VRCPSSr: 4426 case X86::VRCPSSr_Int: 4427 case X86::VRCPSSm: 4428 case X86::VRCPSSm_Int: 4429 case X86::VROUNDSDr: 4430 case X86::VROUNDSDm: 4431 case X86::VROUNDSDr_Int: 4432 case X86::VROUNDSDm_Int: 4433 case X86::VROUNDSSr: 4434 case X86::VROUNDSSm: 4435 case X86::VROUNDSSr_Int: 4436 case X86::VROUNDSSm_Int: 4437 case X86::VRSQRTSSr: 4438 case X86::VRSQRTSSr_Int: 4439 case X86::VRSQRTSSm: 4440 case X86::VRSQRTSSm_Int: 4441 case X86::VSQRTSSr: 4442 case X86::VSQRTSSr_Int: 4443 case X86::VSQRTSSm: 4444 case X86::VSQRTSSm_Int: 4445 case X86::VSQRTSDr: 4446 case X86::VSQRTSDr_Int: 4447 case X86::VSQRTSDm: 4448 case X86::VSQRTSDm_Int: 4449 // AVX-512 4450 case X86::VCVTSI2SSZrr: 4451 case X86::VCVTSI2SSZrm: 4452 case X86::VCVTSI2SSZrr_Int: 4453 case X86::VCVTSI2SSZrrb_Int: 4454 case X86::VCVTSI2SSZrm_Int: 4455 case X86::VCVTSI642SSZrr: 4456 case X86::VCVTSI642SSZrm: 4457 case X86::VCVTSI642SSZrr_Int: 4458 case X86::VCVTSI642SSZrrb_Int: 4459 case X86::VCVTSI642SSZrm_Int: 4460 case X86::VCVTSI2SDZrr: 4461 case X86::VCVTSI2SDZrm: 4462 case X86::VCVTSI2SDZrr_Int: 4463 case X86::VCVTSI2SDZrrb_Int: 4464 case X86::VCVTSI2SDZrm_Int: 4465 case X86::VCVTSI642SDZrr: 4466 case X86::VCVTSI642SDZrm: 4467 case X86::VCVTSI642SDZrr_Int: 4468 case X86::VCVTSI642SDZrrb_Int: 4469 case X86::VCVTSI642SDZrm_Int: 4470 case X86::VCVTUSI2SSZrr: 4471 case X86::VCVTUSI2SSZrm: 4472 case X86::VCVTUSI2SSZrr_Int: 4473 case X86::VCVTUSI2SSZrrb_Int: 4474 case X86::VCVTUSI2SSZrm_Int: 4475 case X86::VCVTUSI642SSZrr: 4476 case X86::VCVTUSI642SSZrm: 4477 case X86::VCVTUSI642SSZrr_Int: 4478 case X86::VCVTUSI642SSZrrb_Int: 4479 case X86::VCVTUSI642SSZrm_Int: 4480 case X86::VCVTUSI2SDZrr: 4481 case X86::VCVTUSI2SDZrm: 4482 case X86::VCVTUSI2SDZrr_Int: 4483 case X86::VCVTUSI2SDZrm_Int: 4484 case X86::VCVTUSI642SDZrr: 4485 case X86::VCVTUSI642SDZrm: 4486 case X86::VCVTUSI642SDZrr_Int: 4487 case X86::VCVTUSI642SDZrrb_Int: 4488 case X86::VCVTUSI642SDZrm_Int: 4489 case X86::VCVTSD2SSZrr: 4490 case X86::VCVTSD2SSZrr_Int: 4491 case X86::VCVTSD2SSZrrb_Int: 4492 case X86::VCVTSD2SSZrm: 4493 case X86::VCVTSD2SSZrm_Int: 4494 case X86::VCVTSS2SDZrr: 4495 case X86::VCVTSS2SDZrr_Int: 4496 case X86::VCVTSS2SDZrrb_Int: 4497 case X86::VCVTSS2SDZrm: 4498 case X86::VCVTSS2SDZrm_Int: 4499 case X86::VGETEXPSDZr: 4500 case X86::VGETEXPSDZrb: 4501 case X86::VGETEXPSDZm: 4502 case X86::VGETEXPSSZr: 4503 case X86::VGETEXPSSZrb: 4504 case X86::VGETEXPSSZm: 4505 case X86::VGETMANTSDZrri: 4506 case X86::VGETMANTSDZrrib: 4507 case X86::VGETMANTSDZrmi: 4508 case X86::VGETMANTSSZrri: 4509 case X86::VGETMANTSSZrrib: 4510 case X86::VGETMANTSSZrmi: 4511 case X86::VRNDSCALESDZr: 4512 case X86::VRNDSCALESDZr_Int: 4513 case X86::VRNDSCALESDZrb_Int: 4514 case X86::VRNDSCALESDZm: 4515 case X86::VRNDSCALESDZm_Int: 4516 case X86::VRNDSCALESSZr: 4517 case X86::VRNDSCALESSZr_Int: 4518 case X86::VRNDSCALESSZrb_Int: 4519 case X86::VRNDSCALESSZm: 4520 case X86::VRNDSCALESSZm_Int: 4521 case X86::VRCP14SDZrr: 4522 case X86::VRCP14SDZrm: 4523 case X86::VRCP14SSZrr: 4524 case X86::VRCP14SSZrm: 4525 case X86::VRCP28SDZr: 4526 case X86::VRCP28SDZrb: 4527 case X86::VRCP28SDZm: 4528 case X86::VRCP28SSZr: 4529 case X86::VRCP28SSZrb: 4530 case X86::VRCP28SSZm: 4531 case X86::VREDUCESSZrmi: 4532 case X86::VREDUCESSZrri: 4533 case X86::VREDUCESSZrrib: 4534 case X86::VRSQRT14SDZrr: 4535 case X86::VRSQRT14SDZrm: 4536 case X86::VRSQRT14SSZrr: 4537 case X86::VRSQRT14SSZrm: 4538 case X86::VRSQRT28SDZr: 4539 case X86::VRSQRT28SDZrb: 4540 case X86::VRSQRT28SDZm: 4541 case X86::VRSQRT28SSZr: 4542 case X86::VRSQRT28SSZrb: 4543 case X86::VRSQRT28SSZm: 4544 case X86::VSQRTSSZr: 4545 case X86::VSQRTSSZr_Int: 4546 case X86::VSQRTSSZrb_Int: 4547 case X86::VSQRTSSZm: 4548 case X86::VSQRTSSZm_Int: 4549 case X86::VSQRTSDZr: 4550 case X86::VSQRTSDZr_Int: 4551 case X86::VSQRTSDZrb_Int: 4552 case X86::VSQRTSDZm: 4553 case X86::VSQRTSDZm_Int: 4554 return true; 4555 } 4556 4557 return false; 4558 } 4559 4560 /// Inform the BreakFalseDeps pass how many idle instructions we would like 4561 /// before certain undef register reads. 4562 /// 4563 /// This catches the VCVTSI2SD family of instructions: 4564 /// 4565 /// vcvtsi2sdq %rax, undef %xmm0, %xmm14 4566 /// 4567 /// We should to be careful *not* to catch VXOR idioms which are presumably 4568 /// handled specially in the pipeline: 4569 /// 4570 /// vxorps undef %xmm1, undef %xmm1, %xmm1 4571 /// 4572 /// Like getPartialRegUpdateClearance, this makes a strong assumption that the 4573 /// high bits that are passed-through are not live. 4574 unsigned 4575 X86InstrInfo::getUndefRegClearance(const MachineInstr &MI, unsigned &OpNum, 4576 const TargetRegisterInfo *TRI) const { 4577 if (!hasUndefRegUpdate(MI.getOpcode())) 4578 return 0; 4579 4580 // Set the OpNum parameter to the first source operand. 4581 OpNum = 1; 4582 4583 const MachineOperand &MO = MI.getOperand(OpNum); 4584 if (MO.isUndef() && TargetRegisterInfo::isPhysicalRegister(MO.getReg())) { 4585 return UndefRegClearance; 4586 } 4587 return 0; 4588 } 4589 4590 void X86InstrInfo::breakPartialRegDependency( 4591 MachineInstr &MI, unsigned OpNum, const TargetRegisterInfo *TRI) const { 4592 unsigned Reg = MI.getOperand(OpNum).getReg(); 4593 // If MI kills this register, the false dependence is already broken. 4594 if (MI.killsRegister(Reg, TRI)) 4595 return; 4596 4597 if (X86::VR128RegClass.contains(Reg)) { 4598 // These instructions are all floating point domain, so xorps is the best 4599 // choice. 4600 unsigned Opc = Subtarget.hasAVX() ? X86::VXORPSrr : X86::XORPSrr; 4601 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(Opc), Reg) 4602 .addReg(Reg, RegState::Undef) 4603 .addReg(Reg, RegState::Undef); 4604 MI.addRegisterKilled(Reg, TRI, true); 4605 } else if (X86::VR256RegClass.contains(Reg)) { 4606 // Use vxorps to clear the full ymm register. 4607 // It wants to read and write the xmm sub-register. 4608 unsigned XReg = TRI->getSubReg(Reg, X86::sub_xmm); 4609 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::VXORPSrr), XReg) 4610 .addReg(XReg, RegState::Undef) 4611 .addReg(XReg, RegState::Undef) 4612 .addReg(Reg, RegState::ImplicitDefine); 4613 MI.addRegisterKilled(Reg, TRI, true); 4614 } else if (X86::GR64RegClass.contains(Reg)) { 4615 // Using XOR32rr because it has shorter encoding and zeros up the upper bits 4616 // as well. 4617 unsigned XReg = TRI->getSubReg(Reg, X86::sub_32bit); 4618 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::XOR32rr), XReg) 4619 .addReg(XReg, RegState::Undef) 4620 .addReg(XReg, RegState::Undef) 4621 .addReg(Reg, RegState::ImplicitDefine); 4622 MI.addRegisterKilled(Reg, TRI, true); 4623 } else if (X86::GR32RegClass.contains(Reg)) { 4624 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::XOR32rr), Reg) 4625 .addReg(Reg, RegState::Undef) 4626 .addReg(Reg, RegState::Undef); 4627 MI.addRegisterKilled(Reg, TRI, true); 4628 } 4629 } 4630 4631 static void addOperands(MachineInstrBuilder &MIB, ArrayRef<MachineOperand> MOs, 4632 int PtrOffset = 0) { 4633 unsigned NumAddrOps = MOs.size(); 4634 4635 if (NumAddrOps < 4) { 4636 // FrameIndex only - add an immediate offset (whether its zero or not). 4637 for (unsigned i = 0; i != NumAddrOps; ++i) 4638 MIB.add(MOs[i]); 4639 addOffset(MIB, PtrOffset); 4640 } else { 4641 // General Memory Addressing - we need to add any offset to an existing 4642 // offset. 4643 assert(MOs.size() == 5 && "Unexpected memory operand list length"); 4644 for (unsigned i = 0; i != NumAddrOps; ++i) { 4645 const MachineOperand &MO = MOs[i]; 4646 if (i == 3 && PtrOffset != 0) { 4647 MIB.addDisp(MO, PtrOffset); 4648 } else { 4649 MIB.add(MO); 4650 } 4651 } 4652 } 4653 } 4654 4655 static MachineInstr *FuseTwoAddrInst(MachineFunction &MF, unsigned Opcode, 4656 ArrayRef<MachineOperand> MOs, 4657 MachineBasicBlock::iterator InsertPt, 4658 MachineInstr &MI, 4659 const TargetInstrInfo &TII) { 4660 // Create the base instruction with the memory operand as the first part. 4661 // Omit the implicit operands, something BuildMI can't do. 4662 MachineInstr *NewMI = 4663 MF.CreateMachineInstr(TII.get(Opcode), MI.getDebugLoc(), true); 4664 MachineInstrBuilder MIB(MF, NewMI); 4665 addOperands(MIB, MOs); 4666 4667 // Loop over the rest of the ri operands, converting them over. 4668 unsigned NumOps = MI.getDesc().getNumOperands() - 2; 4669 for (unsigned i = 0; i != NumOps; ++i) { 4670 MachineOperand &MO = MI.getOperand(i + 2); 4671 MIB.add(MO); 4672 } 4673 for (unsigned i = NumOps + 2, e = MI.getNumOperands(); i != e; ++i) { 4674 MachineOperand &MO = MI.getOperand(i); 4675 MIB.add(MO); 4676 } 4677 4678 MachineBasicBlock *MBB = InsertPt->getParent(); 4679 MBB->insert(InsertPt, NewMI); 4680 4681 return MIB; 4682 } 4683 4684 static MachineInstr *FuseInst(MachineFunction &MF, unsigned Opcode, 4685 unsigned OpNo, ArrayRef<MachineOperand> MOs, 4686 MachineBasicBlock::iterator InsertPt, 4687 MachineInstr &MI, const TargetInstrInfo &TII, 4688 int PtrOffset = 0) { 4689 // Omit the implicit operands, something BuildMI can't do. 4690 MachineInstr *NewMI = 4691 MF.CreateMachineInstr(TII.get(Opcode), MI.getDebugLoc(), true); 4692 MachineInstrBuilder MIB(MF, NewMI); 4693 4694 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 4695 MachineOperand &MO = MI.getOperand(i); 4696 if (i == OpNo) { 4697 assert(MO.isReg() && "Expected to fold into reg operand!"); 4698 addOperands(MIB, MOs, PtrOffset); 4699 } else { 4700 MIB.add(MO); 4701 } 4702 } 4703 4704 MachineBasicBlock *MBB = InsertPt->getParent(); 4705 MBB->insert(InsertPt, NewMI); 4706 4707 return MIB; 4708 } 4709 4710 static MachineInstr *MakeM0Inst(const TargetInstrInfo &TII, unsigned Opcode, 4711 ArrayRef<MachineOperand> MOs, 4712 MachineBasicBlock::iterator InsertPt, 4713 MachineInstr &MI) { 4714 MachineInstrBuilder MIB = BuildMI(*InsertPt->getParent(), InsertPt, 4715 MI.getDebugLoc(), TII.get(Opcode)); 4716 addOperands(MIB, MOs); 4717 return MIB.addImm(0); 4718 } 4719 4720 MachineInstr *X86InstrInfo::foldMemoryOperandCustom( 4721 MachineFunction &MF, MachineInstr &MI, unsigned OpNum, 4722 ArrayRef<MachineOperand> MOs, MachineBasicBlock::iterator InsertPt, 4723 unsigned Size, unsigned Align) const { 4724 switch (MI.getOpcode()) { 4725 case X86::INSERTPSrr: 4726 case X86::VINSERTPSrr: 4727 case X86::VINSERTPSZrr: 4728 // Attempt to convert the load of inserted vector into a fold load 4729 // of a single float. 4730 if (OpNum == 2) { 4731 unsigned Imm = MI.getOperand(MI.getNumOperands() - 1).getImm(); 4732 unsigned ZMask = Imm & 15; 4733 unsigned DstIdx = (Imm >> 4) & 3; 4734 unsigned SrcIdx = (Imm >> 6) & 3; 4735 4736 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 4737 const TargetRegisterClass *RC = getRegClass(MI.getDesc(), OpNum, &RI, MF); 4738 unsigned RCSize = TRI.getRegSizeInBits(*RC) / 8; 4739 if (Size <= RCSize && 4 <= Align) { 4740 int PtrOffset = SrcIdx * 4; 4741 unsigned NewImm = (DstIdx << 4) | ZMask; 4742 unsigned NewOpCode = 4743 (MI.getOpcode() == X86::VINSERTPSZrr) ? X86::VINSERTPSZrm : 4744 (MI.getOpcode() == X86::VINSERTPSrr) ? X86::VINSERTPSrm : 4745 X86::INSERTPSrm; 4746 MachineInstr *NewMI = 4747 FuseInst(MF, NewOpCode, OpNum, MOs, InsertPt, MI, *this, PtrOffset); 4748 NewMI->getOperand(NewMI->getNumOperands() - 1).setImm(NewImm); 4749 return NewMI; 4750 } 4751 } 4752 break; 4753 case X86::MOVHLPSrr: 4754 case X86::VMOVHLPSrr: 4755 case X86::VMOVHLPSZrr: 4756 // Move the upper 64-bits of the second operand to the lower 64-bits. 4757 // To fold the load, adjust the pointer to the upper and use (V)MOVLPS. 4758 // TODO: In most cases AVX doesn't have a 8-byte alignment requirement. 4759 if (OpNum == 2) { 4760 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 4761 const TargetRegisterClass *RC = getRegClass(MI.getDesc(), OpNum, &RI, MF); 4762 unsigned RCSize = TRI.getRegSizeInBits(*RC) / 8; 4763 if (Size <= RCSize && 8 <= Align) { 4764 unsigned NewOpCode = 4765 (MI.getOpcode() == X86::VMOVHLPSZrr) ? X86::VMOVLPSZ128rm : 4766 (MI.getOpcode() == X86::VMOVHLPSrr) ? X86::VMOVLPSrm : 4767 X86::MOVLPSrm; 4768 MachineInstr *NewMI = 4769 FuseInst(MF, NewOpCode, OpNum, MOs, InsertPt, MI, *this, 8); 4770 return NewMI; 4771 } 4772 } 4773 break; 4774 }; 4775 4776 return nullptr; 4777 } 4778 4779 static bool shouldPreventUndefRegUpdateMemFold(MachineFunction &MF, MachineInstr &MI) { 4780 if (MF.getFunction().optForSize() || !hasUndefRegUpdate(MI.getOpcode()) || 4781 !MI.getOperand(1).isReg()) 4782 return false; 4783 4784 // The are two cases we need to handle depending on where in the pipeline 4785 // the folding attempt is being made. 4786 // -Register has the undef flag set. 4787 // -Register is produced by the IMPLICIT_DEF instruction. 4788 4789 if (MI.getOperand(1).isUndef()) 4790 return true; 4791 4792 MachineRegisterInfo &RegInfo = MF.getRegInfo(); 4793 MachineInstr *VRegDef = RegInfo.getUniqueVRegDef(MI.getOperand(1).getReg()); 4794 return VRegDef && VRegDef->isImplicitDef(); 4795 } 4796 4797 4798 MachineInstr *X86InstrInfo::foldMemoryOperandImpl( 4799 MachineFunction &MF, MachineInstr &MI, unsigned OpNum, 4800 ArrayRef<MachineOperand> MOs, MachineBasicBlock::iterator InsertPt, 4801 unsigned Size, unsigned Align, bool AllowCommute) const { 4802 bool isSlowTwoMemOps = Subtarget.slowTwoMemOps(); 4803 bool isTwoAddrFold = false; 4804 4805 // For CPUs that favor the register form of a call or push, 4806 // do not fold loads into calls or pushes, unless optimizing for size 4807 // aggressively. 4808 if (isSlowTwoMemOps && !MF.getFunction().optForMinSize() && 4809 (MI.getOpcode() == X86::CALL32r || MI.getOpcode() == X86::CALL64r || 4810 MI.getOpcode() == X86::PUSH16r || MI.getOpcode() == X86::PUSH32r || 4811 MI.getOpcode() == X86::PUSH64r)) 4812 return nullptr; 4813 4814 // Avoid partial and undef register update stalls unless optimizing for size. 4815 if (!MF.getFunction().optForSize() && 4816 (hasPartialRegUpdate(MI.getOpcode(), Subtarget) || 4817 shouldPreventUndefRegUpdateMemFold(MF, MI))) 4818 return nullptr; 4819 4820 unsigned NumOps = MI.getDesc().getNumOperands(); 4821 bool isTwoAddr = 4822 NumOps > 1 && MI.getDesc().getOperandConstraint(1, MCOI::TIED_TO) != -1; 4823 4824 // FIXME: AsmPrinter doesn't know how to handle 4825 // X86II::MO_GOT_ABSOLUTE_ADDRESS after folding. 4826 if (MI.getOpcode() == X86::ADD32ri && 4827 MI.getOperand(2).getTargetFlags() == X86II::MO_GOT_ABSOLUTE_ADDRESS) 4828 return nullptr; 4829 4830 // GOTTPOFF relocation loads can only be folded into add instructions. 4831 // FIXME: Need to exclude other relocations that only support specific 4832 // instructions. 4833 if (MOs.size() == X86::AddrNumOperands && 4834 MOs[X86::AddrDisp].getTargetFlags() == X86II::MO_GOTTPOFF && 4835 MI.getOpcode() != X86::ADD64rr) 4836 return nullptr; 4837 4838 MachineInstr *NewMI = nullptr; 4839 4840 // Attempt to fold any custom cases we have. 4841 if (MachineInstr *CustomMI = 4842 foldMemoryOperandCustom(MF, MI, OpNum, MOs, InsertPt, Size, Align)) 4843 return CustomMI; 4844 4845 const X86MemoryFoldTableEntry *I = nullptr; 4846 4847 // Folding a memory location into the two-address part of a two-address 4848 // instruction is different than folding it other places. It requires 4849 // replacing the *two* registers with the memory location. 4850 if (isTwoAddr && NumOps >= 2 && OpNum < 2 && MI.getOperand(0).isReg() && 4851 MI.getOperand(1).isReg() && 4852 MI.getOperand(0).getReg() == MI.getOperand(1).getReg()) { 4853 I = lookupTwoAddrFoldTable(MI.getOpcode()); 4854 isTwoAddrFold = true; 4855 } else { 4856 if (OpNum == 0) { 4857 if (MI.getOpcode() == X86::MOV32r0) { 4858 NewMI = MakeM0Inst(*this, X86::MOV32mi, MOs, InsertPt, MI); 4859 if (NewMI) 4860 return NewMI; 4861 } 4862 } 4863 4864 I = lookupFoldTable(MI.getOpcode(), OpNum); 4865 } 4866 4867 if (I != nullptr) { 4868 unsigned Opcode = I->DstOp; 4869 unsigned MinAlign = (I->Flags & TB_ALIGN_MASK) >> TB_ALIGN_SHIFT; 4870 if (Align < MinAlign) 4871 return nullptr; 4872 bool NarrowToMOV32rm = false; 4873 if (Size) { 4874 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 4875 const TargetRegisterClass *RC = getRegClass(MI.getDesc(), OpNum, 4876 &RI, MF); 4877 unsigned RCSize = TRI.getRegSizeInBits(*RC) / 8; 4878 if (Size < RCSize) { 4879 // Check if it's safe to fold the load. If the size of the object is 4880 // narrower than the load width, then it's not. 4881 if (Opcode != X86::MOV64rm || RCSize != 8 || Size != 4) 4882 return nullptr; 4883 // If this is a 64-bit load, but the spill slot is 32, then we can do 4884 // a 32-bit load which is implicitly zero-extended. This likely is 4885 // due to live interval analysis remat'ing a load from stack slot. 4886 if (MI.getOperand(0).getSubReg() || MI.getOperand(1).getSubReg()) 4887 return nullptr; 4888 Opcode = X86::MOV32rm; 4889 NarrowToMOV32rm = true; 4890 } 4891 } 4892 4893 if (isTwoAddrFold) 4894 NewMI = FuseTwoAddrInst(MF, Opcode, MOs, InsertPt, MI, *this); 4895 else 4896 NewMI = FuseInst(MF, Opcode, OpNum, MOs, InsertPt, MI, *this); 4897 4898 if (NarrowToMOV32rm) { 4899 // If this is the special case where we use a MOV32rm to load a 32-bit 4900 // value and zero-extend the top bits. Change the destination register 4901 // to a 32-bit one. 4902 unsigned DstReg = NewMI->getOperand(0).getReg(); 4903 if (TargetRegisterInfo::isPhysicalRegister(DstReg)) 4904 NewMI->getOperand(0).setReg(RI.getSubReg(DstReg, X86::sub_32bit)); 4905 else 4906 NewMI->getOperand(0).setSubReg(X86::sub_32bit); 4907 } 4908 return NewMI; 4909 } 4910 4911 // If the instruction and target operand are commutable, commute the 4912 // instruction and try again. 4913 if (AllowCommute) { 4914 unsigned CommuteOpIdx1 = OpNum, CommuteOpIdx2 = CommuteAnyOperandIndex; 4915 if (findCommutedOpIndices(MI, CommuteOpIdx1, CommuteOpIdx2)) { 4916 bool HasDef = MI.getDesc().getNumDefs(); 4917 unsigned Reg0 = HasDef ? MI.getOperand(0).getReg() : 0; 4918 unsigned Reg1 = MI.getOperand(CommuteOpIdx1).getReg(); 4919 unsigned Reg2 = MI.getOperand(CommuteOpIdx2).getReg(); 4920 bool Tied1 = 4921 0 == MI.getDesc().getOperandConstraint(CommuteOpIdx1, MCOI::TIED_TO); 4922 bool Tied2 = 4923 0 == MI.getDesc().getOperandConstraint(CommuteOpIdx2, MCOI::TIED_TO); 4924 4925 // If either of the commutable operands are tied to the destination 4926 // then we can not commute + fold. 4927 if ((HasDef && Reg0 == Reg1 && Tied1) || 4928 (HasDef && Reg0 == Reg2 && Tied2)) 4929 return nullptr; 4930 4931 MachineInstr *CommutedMI = 4932 commuteInstruction(MI, false, CommuteOpIdx1, CommuteOpIdx2); 4933 if (!CommutedMI) { 4934 // Unable to commute. 4935 return nullptr; 4936 } 4937 if (CommutedMI != &MI) { 4938 // New instruction. We can't fold from this. 4939 CommutedMI->eraseFromParent(); 4940 return nullptr; 4941 } 4942 4943 // Attempt to fold with the commuted version of the instruction. 4944 NewMI = foldMemoryOperandImpl(MF, MI, CommuteOpIdx2, MOs, InsertPt, 4945 Size, Align, /*AllowCommute=*/false); 4946 if (NewMI) 4947 return NewMI; 4948 4949 // Folding failed again - undo the commute before returning. 4950 MachineInstr *UncommutedMI = 4951 commuteInstruction(MI, false, CommuteOpIdx1, CommuteOpIdx2); 4952 if (!UncommutedMI) { 4953 // Unable to commute. 4954 return nullptr; 4955 } 4956 if (UncommutedMI != &MI) { 4957 // New instruction. It doesn't need to be kept. 4958 UncommutedMI->eraseFromParent(); 4959 return nullptr; 4960 } 4961 4962 // Return here to prevent duplicate fuse failure report. 4963 return nullptr; 4964 } 4965 } 4966 4967 // No fusion 4968 if (PrintFailedFusing && !MI.isCopy()) 4969 dbgs() << "We failed to fuse operand " << OpNum << " in " << MI; 4970 return nullptr; 4971 } 4972 4973 MachineInstr * 4974 X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, 4975 ArrayRef<unsigned> Ops, 4976 MachineBasicBlock::iterator InsertPt, 4977 int FrameIndex, LiveIntervals *LIS) const { 4978 // Check switch flag 4979 if (NoFusing) 4980 return nullptr; 4981 4982 // Avoid partial and undef register update stalls unless optimizing for size. 4983 if (!MF.getFunction().optForSize() && 4984 (hasPartialRegUpdate(MI.getOpcode(), Subtarget) || 4985 shouldPreventUndefRegUpdateMemFold(MF, MI))) 4986 return nullptr; 4987 4988 // Don't fold subreg spills, or reloads that use a high subreg. 4989 for (auto Op : Ops) { 4990 MachineOperand &MO = MI.getOperand(Op); 4991 auto SubReg = MO.getSubReg(); 4992 if (SubReg && (MO.isDef() || SubReg == X86::sub_8bit_hi)) 4993 return nullptr; 4994 } 4995 4996 const MachineFrameInfo &MFI = MF.getFrameInfo(); 4997 unsigned Size = MFI.getObjectSize(FrameIndex); 4998 unsigned Alignment = MFI.getObjectAlignment(FrameIndex); 4999 // If the function stack isn't realigned we don't want to fold instructions 5000 // that need increased alignment. 5001 if (!RI.needsStackRealignment(MF)) 5002 Alignment = 5003 std::min(Alignment, Subtarget.getFrameLowering()->getStackAlignment()); 5004 if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) { 5005 unsigned NewOpc = 0; 5006 unsigned RCSize = 0; 5007 switch (MI.getOpcode()) { 5008 default: return nullptr; 5009 case X86::TEST8rr: NewOpc = X86::CMP8ri; RCSize = 1; break; 5010 case X86::TEST16rr: NewOpc = X86::CMP16ri8; RCSize = 2; break; 5011 case X86::TEST32rr: NewOpc = X86::CMP32ri8; RCSize = 4; break; 5012 case X86::TEST64rr: NewOpc = X86::CMP64ri8; RCSize = 8; break; 5013 } 5014 // Check if it's safe to fold the load. If the size of the object is 5015 // narrower than the load width, then it's not. 5016 if (Size < RCSize) 5017 return nullptr; 5018 // Change to CMPXXri r, 0 first. 5019 MI.setDesc(get(NewOpc)); 5020 MI.getOperand(1).ChangeToImmediate(0); 5021 } else if (Ops.size() != 1) 5022 return nullptr; 5023 5024 return foldMemoryOperandImpl(MF, MI, Ops[0], 5025 MachineOperand::CreateFI(FrameIndex), InsertPt, 5026 Size, Alignment, /*AllowCommute=*/true); 5027 } 5028 5029 /// Check if \p LoadMI is a partial register load that we can't fold into \p MI 5030 /// because the latter uses contents that wouldn't be defined in the folded 5031 /// version. For instance, this transformation isn't legal: 5032 /// movss (%rdi), %xmm0 5033 /// addps %xmm0, %xmm0 5034 /// -> 5035 /// addps (%rdi), %xmm0 5036 /// 5037 /// But this one is: 5038 /// movss (%rdi), %xmm0 5039 /// addss %xmm0, %xmm0 5040 /// -> 5041 /// addss (%rdi), %xmm0 5042 /// 5043 static bool isNonFoldablePartialRegisterLoad(const MachineInstr &LoadMI, 5044 const MachineInstr &UserMI, 5045 const MachineFunction &MF) { 5046 unsigned Opc = LoadMI.getOpcode(); 5047 unsigned UserOpc = UserMI.getOpcode(); 5048 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 5049 const TargetRegisterClass *RC = 5050 MF.getRegInfo().getRegClass(LoadMI.getOperand(0).getReg()); 5051 unsigned RegSize = TRI.getRegSizeInBits(*RC); 5052 5053 if ((Opc == X86::MOVSSrm || Opc == X86::VMOVSSrm || Opc == X86::VMOVSSZrm) && 5054 RegSize > 32) { 5055 // These instructions only load 32 bits, we can't fold them if the 5056 // destination register is wider than 32 bits (4 bytes), and its user 5057 // instruction isn't scalar (SS). 5058 switch (UserOpc) { 5059 case X86::ADDSSrr_Int: case X86::VADDSSrr_Int: case X86::VADDSSZrr_Int: 5060 case X86::CMPSSrr_Int: case X86::VCMPSSrr_Int: case X86::VCMPSSZrr_Int: 5061 case X86::DIVSSrr_Int: case X86::VDIVSSrr_Int: case X86::VDIVSSZrr_Int: 5062 case X86::MAXSSrr_Int: case X86::VMAXSSrr_Int: case X86::VMAXSSZrr_Int: 5063 case X86::MINSSrr_Int: case X86::VMINSSrr_Int: case X86::VMINSSZrr_Int: 5064 case X86::MULSSrr_Int: case X86::VMULSSrr_Int: case X86::VMULSSZrr_Int: 5065 case X86::SUBSSrr_Int: case X86::VSUBSSrr_Int: case X86::VSUBSSZrr_Int: 5066 case X86::VADDSSZrr_Intk: case X86::VADDSSZrr_Intkz: 5067 case X86::VDIVSSZrr_Intk: case X86::VDIVSSZrr_Intkz: 5068 case X86::VMAXSSZrr_Intk: case X86::VMAXSSZrr_Intkz: 5069 case X86::VMINSSZrr_Intk: case X86::VMINSSZrr_Intkz: 5070 case X86::VMULSSZrr_Intk: case X86::VMULSSZrr_Intkz: 5071 case X86::VSUBSSZrr_Intk: case X86::VSUBSSZrr_Intkz: 5072 case X86::VFMADDSS4rr_Int: case X86::VFNMADDSS4rr_Int: 5073 case X86::VFMSUBSS4rr_Int: case X86::VFNMSUBSS4rr_Int: 5074 case X86::VFMADD132SSr_Int: case X86::VFNMADD132SSr_Int: 5075 case X86::VFMADD213SSr_Int: case X86::VFNMADD213SSr_Int: 5076 case X86::VFMADD231SSr_Int: case X86::VFNMADD231SSr_Int: 5077 case X86::VFMSUB132SSr_Int: case X86::VFNMSUB132SSr_Int: 5078 case X86::VFMSUB213SSr_Int: case X86::VFNMSUB213SSr_Int: 5079 case X86::VFMSUB231SSr_Int: case X86::VFNMSUB231SSr_Int: 5080 case X86::VFMADD132SSZr_Int: case X86::VFNMADD132SSZr_Int: 5081 case X86::VFMADD213SSZr_Int: case X86::VFNMADD213SSZr_Int: 5082 case X86::VFMADD231SSZr_Int: case X86::VFNMADD231SSZr_Int: 5083 case X86::VFMSUB132SSZr_Int: case X86::VFNMSUB132SSZr_Int: 5084 case X86::VFMSUB213SSZr_Int: case X86::VFNMSUB213SSZr_Int: 5085 case X86::VFMSUB231SSZr_Int: case X86::VFNMSUB231SSZr_Int: 5086 case X86::VFMADD132SSZr_Intk: case X86::VFNMADD132SSZr_Intk: 5087 case X86::VFMADD213SSZr_Intk: case X86::VFNMADD213SSZr_Intk: 5088 case X86::VFMADD231SSZr_Intk: case X86::VFNMADD231SSZr_Intk: 5089 case X86::VFMSUB132SSZr_Intk: case X86::VFNMSUB132SSZr_Intk: 5090 case X86::VFMSUB213SSZr_Intk: case X86::VFNMSUB213SSZr_Intk: 5091 case X86::VFMSUB231SSZr_Intk: case X86::VFNMSUB231SSZr_Intk: 5092 case X86::VFMADD132SSZr_Intkz: case X86::VFNMADD132SSZr_Intkz: 5093 case X86::VFMADD213SSZr_Intkz: case X86::VFNMADD213SSZr_Intkz: 5094 case X86::VFMADD231SSZr_Intkz: case X86::VFNMADD231SSZr_Intkz: 5095 case X86::VFMSUB132SSZr_Intkz: case X86::VFNMSUB132SSZr_Intkz: 5096 case X86::VFMSUB213SSZr_Intkz: case X86::VFNMSUB213SSZr_Intkz: 5097 case X86::VFMSUB231SSZr_Intkz: case X86::VFNMSUB231SSZr_Intkz: 5098 return false; 5099 default: 5100 return true; 5101 } 5102 } 5103 5104 if ((Opc == X86::MOVSDrm || Opc == X86::VMOVSDrm || Opc == X86::VMOVSDZrm) && 5105 RegSize > 64) { 5106 // These instructions only load 64 bits, we can't fold them if the 5107 // destination register is wider than 64 bits (8 bytes), and its user 5108 // instruction isn't scalar (SD). 5109 switch (UserOpc) { 5110 case X86::ADDSDrr_Int: case X86::VADDSDrr_Int: case X86::VADDSDZrr_Int: 5111 case X86::CMPSDrr_Int: case X86::VCMPSDrr_Int: case X86::VCMPSDZrr_Int: 5112 case X86::DIVSDrr_Int: case X86::VDIVSDrr_Int: case X86::VDIVSDZrr_Int: 5113 case X86::MAXSDrr_Int: case X86::VMAXSDrr_Int: case X86::VMAXSDZrr_Int: 5114 case X86::MINSDrr_Int: case X86::VMINSDrr_Int: case X86::VMINSDZrr_Int: 5115 case X86::MULSDrr_Int: case X86::VMULSDrr_Int: case X86::VMULSDZrr_Int: 5116 case X86::SUBSDrr_Int: case X86::VSUBSDrr_Int: case X86::VSUBSDZrr_Int: 5117 case X86::VADDSDZrr_Intk: case X86::VADDSDZrr_Intkz: 5118 case X86::VDIVSDZrr_Intk: case X86::VDIVSDZrr_Intkz: 5119 case X86::VMAXSDZrr_Intk: case X86::VMAXSDZrr_Intkz: 5120 case X86::VMINSDZrr_Intk: case X86::VMINSDZrr_Intkz: 5121 case X86::VMULSDZrr_Intk: case X86::VMULSDZrr_Intkz: 5122 case X86::VSUBSDZrr_Intk: case X86::VSUBSDZrr_Intkz: 5123 case X86::VFMADDSD4rr_Int: case X86::VFNMADDSD4rr_Int: 5124 case X86::VFMSUBSD4rr_Int: case X86::VFNMSUBSD4rr_Int: 5125 case X86::VFMADD132SDr_Int: case X86::VFNMADD132SDr_Int: 5126 case X86::VFMADD213SDr_Int: case X86::VFNMADD213SDr_Int: 5127 case X86::VFMADD231SDr_Int: case X86::VFNMADD231SDr_Int: 5128 case X86::VFMSUB132SDr_Int: case X86::VFNMSUB132SDr_Int: 5129 case X86::VFMSUB213SDr_Int: case X86::VFNMSUB213SDr_Int: 5130 case X86::VFMSUB231SDr_Int: case X86::VFNMSUB231SDr_Int: 5131 case X86::VFMADD132SDZr_Int: case X86::VFNMADD132SDZr_Int: 5132 case X86::VFMADD213SDZr_Int: case X86::VFNMADD213SDZr_Int: 5133 case X86::VFMADD231SDZr_Int: case X86::VFNMADD231SDZr_Int: 5134 case X86::VFMSUB132SDZr_Int: case X86::VFNMSUB132SDZr_Int: 5135 case X86::VFMSUB213SDZr_Int: case X86::VFNMSUB213SDZr_Int: 5136 case X86::VFMSUB231SDZr_Int: case X86::VFNMSUB231SDZr_Int: 5137 case X86::VFMADD132SDZr_Intk: case X86::VFNMADD132SDZr_Intk: 5138 case X86::VFMADD213SDZr_Intk: case X86::VFNMADD213SDZr_Intk: 5139 case X86::VFMADD231SDZr_Intk: case X86::VFNMADD231SDZr_Intk: 5140 case X86::VFMSUB132SDZr_Intk: case X86::VFNMSUB132SDZr_Intk: 5141 case X86::VFMSUB213SDZr_Intk: case X86::VFNMSUB213SDZr_Intk: 5142 case X86::VFMSUB231SDZr_Intk: case X86::VFNMSUB231SDZr_Intk: 5143 case X86::VFMADD132SDZr_Intkz: case X86::VFNMADD132SDZr_Intkz: 5144 case X86::VFMADD213SDZr_Intkz: case X86::VFNMADD213SDZr_Intkz: 5145 case X86::VFMADD231SDZr_Intkz: case X86::VFNMADD231SDZr_Intkz: 5146 case X86::VFMSUB132SDZr_Intkz: case X86::VFNMSUB132SDZr_Intkz: 5147 case X86::VFMSUB213SDZr_Intkz: case X86::VFNMSUB213SDZr_Intkz: 5148 case X86::VFMSUB231SDZr_Intkz: case X86::VFNMSUB231SDZr_Intkz: 5149 return false; 5150 default: 5151 return true; 5152 } 5153 } 5154 5155 return false; 5156 } 5157 5158 MachineInstr *X86InstrInfo::foldMemoryOperandImpl( 5159 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops, 5160 MachineBasicBlock::iterator InsertPt, MachineInstr &LoadMI, 5161 LiveIntervals *LIS) const { 5162 5163 // TODO: Support the case where LoadMI loads a wide register, but MI 5164 // only uses a subreg. 5165 for (auto Op : Ops) { 5166 if (MI.getOperand(Op).getSubReg()) 5167 return nullptr; 5168 } 5169 5170 // If loading from a FrameIndex, fold directly from the FrameIndex. 5171 unsigned NumOps = LoadMI.getDesc().getNumOperands(); 5172 int FrameIndex; 5173 if (isLoadFromStackSlot(LoadMI, FrameIndex)) { 5174 if (isNonFoldablePartialRegisterLoad(LoadMI, MI, MF)) 5175 return nullptr; 5176 return foldMemoryOperandImpl(MF, MI, Ops, InsertPt, FrameIndex, LIS); 5177 } 5178 5179 // Check switch flag 5180 if (NoFusing) return nullptr; 5181 5182 // Avoid partial and undef register update stalls unless optimizing for size. 5183 if (!MF.getFunction().optForSize() && 5184 (hasPartialRegUpdate(MI.getOpcode(), Subtarget) || 5185 shouldPreventUndefRegUpdateMemFold(MF, MI))) 5186 return nullptr; 5187 5188 // Determine the alignment of the load. 5189 unsigned Alignment = 0; 5190 if (LoadMI.hasOneMemOperand()) 5191 Alignment = (*LoadMI.memoperands_begin())->getAlignment(); 5192 else 5193 switch (LoadMI.getOpcode()) { 5194 case X86::AVX512_512_SET0: 5195 case X86::AVX512_512_SETALLONES: 5196 Alignment = 64; 5197 break; 5198 case X86::AVX2_SETALLONES: 5199 case X86::AVX1_SETALLONES: 5200 case X86::AVX_SET0: 5201 case X86::AVX512_256_SET0: 5202 Alignment = 32; 5203 break; 5204 case X86::V_SET0: 5205 case X86::V_SETALLONES: 5206 case X86::AVX512_128_SET0: 5207 Alignment = 16; 5208 break; 5209 case X86::MMX_SET0: 5210 case X86::FsFLD0SD: 5211 case X86::AVX512_FsFLD0SD: 5212 Alignment = 8; 5213 break; 5214 case X86::FsFLD0SS: 5215 case X86::AVX512_FsFLD0SS: 5216 Alignment = 4; 5217 break; 5218 default: 5219 return nullptr; 5220 } 5221 if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) { 5222 unsigned NewOpc = 0; 5223 switch (MI.getOpcode()) { 5224 default: return nullptr; 5225 case X86::TEST8rr: NewOpc = X86::CMP8ri; break; 5226 case X86::TEST16rr: NewOpc = X86::CMP16ri8; break; 5227 case X86::TEST32rr: NewOpc = X86::CMP32ri8; break; 5228 case X86::TEST64rr: NewOpc = X86::CMP64ri8; break; 5229 } 5230 // Change to CMPXXri r, 0 first. 5231 MI.setDesc(get(NewOpc)); 5232 MI.getOperand(1).ChangeToImmediate(0); 5233 } else if (Ops.size() != 1) 5234 return nullptr; 5235 5236 // Make sure the subregisters match. 5237 // Otherwise we risk changing the size of the load. 5238 if (LoadMI.getOperand(0).getSubReg() != MI.getOperand(Ops[0]).getSubReg()) 5239 return nullptr; 5240 5241 SmallVector<MachineOperand,X86::AddrNumOperands> MOs; 5242 switch (LoadMI.getOpcode()) { 5243 case X86::MMX_SET0: 5244 case X86::V_SET0: 5245 case X86::V_SETALLONES: 5246 case X86::AVX2_SETALLONES: 5247 case X86::AVX1_SETALLONES: 5248 case X86::AVX_SET0: 5249 case X86::AVX512_128_SET0: 5250 case X86::AVX512_256_SET0: 5251 case X86::AVX512_512_SET0: 5252 case X86::AVX512_512_SETALLONES: 5253 case X86::FsFLD0SD: 5254 case X86::AVX512_FsFLD0SD: 5255 case X86::FsFLD0SS: 5256 case X86::AVX512_FsFLD0SS: { 5257 // Folding a V_SET0 or V_SETALLONES as a load, to ease register pressure. 5258 // Create a constant-pool entry and operands to load from it. 5259 5260 // Medium and large mode can't fold loads this way. 5261 if (MF.getTarget().getCodeModel() != CodeModel::Small && 5262 MF.getTarget().getCodeModel() != CodeModel::Kernel) 5263 return nullptr; 5264 5265 // x86-32 PIC requires a PIC base register for constant pools. 5266 unsigned PICBase = 0; 5267 if (MF.getTarget().isPositionIndependent()) { 5268 if (Subtarget.is64Bit()) 5269 PICBase = X86::RIP; 5270 else 5271 // FIXME: PICBase = getGlobalBaseReg(&MF); 5272 // This doesn't work for several reasons. 5273 // 1. GlobalBaseReg may have been spilled. 5274 // 2. It may not be live at MI. 5275 return nullptr; 5276 } 5277 5278 // Create a constant-pool entry. 5279 MachineConstantPool &MCP = *MF.getConstantPool(); 5280 Type *Ty; 5281 unsigned Opc = LoadMI.getOpcode(); 5282 if (Opc == X86::FsFLD0SS || Opc == X86::AVX512_FsFLD0SS) 5283 Ty = Type::getFloatTy(MF.getFunction().getContext()); 5284 else if (Opc == X86::FsFLD0SD || Opc == X86::AVX512_FsFLD0SD) 5285 Ty = Type::getDoubleTy(MF.getFunction().getContext()); 5286 else if (Opc == X86::AVX512_512_SET0 || Opc == X86::AVX512_512_SETALLONES) 5287 Ty = VectorType::get(Type::getInt32Ty(MF.getFunction().getContext()),16); 5288 else if (Opc == X86::AVX2_SETALLONES || Opc == X86::AVX_SET0 || 5289 Opc == X86::AVX512_256_SET0 || Opc == X86::AVX1_SETALLONES) 5290 Ty = VectorType::get(Type::getInt32Ty(MF.getFunction().getContext()), 8); 5291 else if (Opc == X86::MMX_SET0) 5292 Ty = VectorType::get(Type::getInt32Ty(MF.getFunction().getContext()), 2); 5293 else 5294 Ty = VectorType::get(Type::getInt32Ty(MF.getFunction().getContext()), 4); 5295 5296 bool IsAllOnes = (Opc == X86::V_SETALLONES || Opc == X86::AVX2_SETALLONES || 5297 Opc == X86::AVX512_512_SETALLONES || 5298 Opc == X86::AVX1_SETALLONES); 5299 const Constant *C = IsAllOnes ? Constant::getAllOnesValue(Ty) : 5300 Constant::getNullValue(Ty); 5301 unsigned CPI = MCP.getConstantPoolIndex(C, Alignment); 5302 5303 // Create operands to load from the constant pool entry. 5304 MOs.push_back(MachineOperand::CreateReg(PICBase, false)); 5305 MOs.push_back(MachineOperand::CreateImm(1)); 5306 MOs.push_back(MachineOperand::CreateReg(0, false)); 5307 MOs.push_back(MachineOperand::CreateCPI(CPI, 0)); 5308 MOs.push_back(MachineOperand::CreateReg(0, false)); 5309 break; 5310 } 5311 default: { 5312 if (isNonFoldablePartialRegisterLoad(LoadMI, MI, MF)) 5313 return nullptr; 5314 5315 // Folding a normal load. Just copy the load's address operands. 5316 MOs.append(LoadMI.operands_begin() + NumOps - X86::AddrNumOperands, 5317 LoadMI.operands_begin() + NumOps); 5318 break; 5319 } 5320 } 5321 return foldMemoryOperandImpl(MF, MI, Ops[0], MOs, InsertPt, 5322 /*Size=*/0, Alignment, /*AllowCommute=*/true); 5323 } 5324 5325 bool X86InstrInfo::unfoldMemoryOperand( 5326 MachineFunction &MF, MachineInstr &MI, unsigned Reg, bool UnfoldLoad, 5327 bool UnfoldStore, SmallVectorImpl<MachineInstr *> &NewMIs) const { 5328 const X86MemoryFoldTableEntry *I = lookupUnfoldTable(MI.getOpcode()); 5329 if (I == nullptr) 5330 return false; 5331 unsigned Opc = I->DstOp; 5332 unsigned Index = I->Flags & TB_INDEX_MASK; 5333 bool FoldedLoad = I->Flags & TB_FOLDED_LOAD; 5334 bool FoldedStore = I->Flags & TB_FOLDED_STORE; 5335 if (UnfoldLoad && !FoldedLoad) 5336 return false; 5337 UnfoldLoad &= FoldedLoad; 5338 if (UnfoldStore && !FoldedStore) 5339 return false; 5340 UnfoldStore &= FoldedStore; 5341 5342 const MCInstrDesc &MCID = get(Opc); 5343 const TargetRegisterClass *RC = getRegClass(MCID, Index, &RI, MF); 5344 // TODO: Check if 32-byte or greater accesses are slow too? 5345 if (!MI.hasOneMemOperand() && RC == &X86::VR128RegClass && 5346 Subtarget.isUnalignedMem16Slow()) 5347 // Without memoperands, loadRegFromAddr and storeRegToStackSlot will 5348 // conservatively assume the address is unaligned. That's bad for 5349 // performance. 5350 return false; 5351 SmallVector<MachineOperand, X86::AddrNumOperands> AddrOps; 5352 SmallVector<MachineOperand,2> BeforeOps; 5353 SmallVector<MachineOperand,2> AfterOps; 5354 SmallVector<MachineOperand,4> ImpOps; 5355 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 5356 MachineOperand &Op = MI.getOperand(i); 5357 if (i >= Index && i < Index + X86::AddrNumOperands) 5358 AddrOps.push_back(Op); 5359 else if (Op.isReg() && Op.isImplicit()) 5360 ImpOps.push_back(Op); 5361 else if (i < Index) 5362 BeforeOps.push_back(Op); 5363 else if (i > Index) 5364 AfterOps.push_back(Op); 5365 } 5366 5367 // Emit the load instruction. 5368 if (UnfoldLoad) { 5369 std::pair<MachineInstr::mmo_iterator, MachineInstr::mmo_iterator> MMOs = 5370 MF.extractLoadMemRefs(MI.memoperands_begin(), MI.memoperands_end()); 5371 loadRegFromAddr(MF, Reg, AddrOps, RC, MMOs.first, MMOs.second, NewMIs); 5372 if (UnfoldStore) { 5373 // Address operands cannot be marked isKill. 5374 for (unsigned i = 1; i != 1 + X86::AddrNumOperands; ++i) { 5375 MachineOperand &MO = NewMIs[0]->getOperand(i); 5376 if (MO.isReg()) 5377 MO.setIsKill(false); 5378 } 5379 } 5380 } 5381 5382 // Emit the data processing instruction. 5383 MachineInstr *DataMI = MF.CreateMachineInstr(MCID, MI.getDebugLoc(), true); 5384 MachineInstrBuilder MIB(MF, DataMI); 5385 5386 if (FoldedStore) 5387 MIB.addReg(Reg, RegState::Define); 5388 for (MachineOperand &BeforeOp : BeforeOps) 5389 MIB.add(BeforeOp); 5390 if (FoldedLoad) 5391 MIB.addReg(Reg); 5392 for (MachineOperand &AfterOp : AfterOps) 5393 MIB.add(AfterOp); 5394 for (MachineOperand &ImpOp : ImpOps) { 5395 MIB.addReg(ImpOp.getReg(), 5396 getDefRegState(ImpOp.isDef()) | 5397 RegState::Implicit | 5398 getKillRegState(ImpOp.isKill()) | 5399 getDeadRegState(ImpOp.isDead()) | 5400 getUndefRegState(ImpOp.isUndef())); 5401 } 5402 // Change CMP32ri r, 0 back to TEST32rr r, r, etc. 5403 switch (DataMI->getOpcode()) { 5404 default: break; 5405 case X86::CMP64ri32: 5406 case X86::CMP64ri8: 5407 case X86::CMP32ri: 5408 case X86::CMP32ri8: 5409 case X86::CMP16ri: 5410 case X86::CMP16ri8: 5411 case X86::CMP8ri: { 5412 MachineOperand &MO0 = DataMI->getOperand(0); 5413 MachineOperand &MO1 = DataMI->getOperand(1); 5414 if (MO1.getImm() == 0) { 5415 unsigned NewOpc; 5416 switch (DataMI->getOpcode()) { 5417 default: llvm_unreachable("Unreachable!"); 5418 case X86::CMP64ri8: 5419 case X86::CMP64ri32: NewOpc = X86::TEST64rr; break; 5420 case X86::CMP32ri8: 5421 case X86::CMP32ri: NewOpc = X86::TEST32rr; break; 5422 case X86::CMP16ri8: 5423 case X86::CMP16ri: NewOpc = X86::TEST16rr; break; 5424 case X86::CMP8ri: NewOpc = X86::TEST8rr; break; 5425 } 5426 DataMI->setDesc(get(NewOpc)); 5427 MO1.ChangeToRegister(MO0.getReg(), false); 5428 } 5429 } 5430 } 5431 NewMIs.push_back(DataMI); 5432 5433 // Emit the store instruction. 5434 if (UnfoldStore) { 5435 const TargetRegisterClass *DstRC = getRegClass(MCID, 0, &RI, MF); 5436 std::pair<MachineInstr::mmo_iterator, MachineInstr::mmo_iterator> MMOs = 5437 MF.extractStoreMemRefs(MI.memoperands_begin(), MI.memoperands_end()); 5438 storeRegToAddr(MF, Reg, true, AddrOps, DstRC, MMOs.first, MMOs.second, NewMIs); 5439 } 5440 5441 return true; 5442 } 5443 5444 bool 5445 X86InstrInfo::unfoldMemoryOperand(SelectionDAG &DAG, SDNode *N, 5446 SmallVectorImpl<SDNode*> &NewNodes) const { 5447 if (!N->isMachineOpcode()) 5448 return false; 5449 5450 const X86MemoryFoldTableEntry *I = lookupUnfoldTable(N->getMachineOpcode()); 5451 if (I == nullptr) 5452 return false; 5453 unsigned Opc = I->DstOp; 5454 unsigned Index = I->Flags & TB_INDEX_MASK; 5455 bool FoldedLoad = I->Flags & TB_FOLDED_LOAD; 5456 bool FoldedStore = I->Flags & TB_FOLDED_STORE; 5457 const MCInstrDesc &MCID = get(Opc); 5458 MachineFunction &MF = DAG.getMachineFunction(); 5459 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 5460 const TargetRegisterClass *RC = getRegClass(MCID, Index, &RI, MF); 5461 unsigned NumDefs = MCID.NumDefs; 5462 std::vector<SDValue> AddrOps; 5463 std::vector<SDValue> BeforeOps; 5464 std::vector<SDValue> AfterOps; 5465 SDLoc dl(N); 5466 unsigned NumOps = N->getNumOperands(); 5467 for (unsigned i = 0; i != NumOps-1; ++i) { 5468 SDValue Op = N->getOperand(i); 5469 if (i >= Index-NumDefs && i < Index-NumDefs + X86::AddrNumOperands) 5470 AddrOps.push_back(Op); 5471 else if (i < Index-NumDefs) 5472 BeforeOps.push_back(Op); 5473 else if (i > Index-NumDefs) 5474 AfterOps.push_back(Op); 5475 } 5476 SDValue Chain = N->getOperand(NumOps-1); 5477 AddrOps.push_back(Chain); 5478 5479 // Emit the load instruction. 5480 SDNode *Load = nullptr; 5481 if (FoldedLoad) { 5482 EVT VT = *TRI.legalclasstypes_begin(*RC); 5483 std::pair<MachineInstr::mmo_iterator, 5484 MachineInstr::mmo_iterator> MMOs = 5485 MF.extractLoadMemRefs(cast<MachineSDNode>(N)->memoperands_begin(), 5486 cast<MachineSDNode>(N)->memoperands_end()); 5487 if (!(*MMOs.first) && 5488 RC == &X86::VR128RegClass && 5489 Subtarget.isUnalignedMem16Slow()) 5490 // Do not introduce a slow unaligned load. 5491 return false; 5492 // FIXME: If a VR128 can have size 32, we should be checking if a 32-byte 5493 // memory access is slow above. 5494 unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16); 5495 bool isAligned = (*MMOs.first) && 5496 (*MMOs.first)->getAlignment() >= Alignment; 5497 Load = DAG.getMachineNode(getLoadRegOpcode(0, RC, isAligned, Subtarget), dl, 5498 VT, MVT::Other, AddrOps); 5499 NewNodes.push_back(Load); 5500 5501 // Preserve memory reference information. 5502 cast<MachineSDNode>(Load)->setMemRefs(MMOs.first, MMOs.second); 5503 } 5504 5505 // Emit the data processing instruction. 5506 std::vector<EVT> VTs; 5507 const TargetRegisterClass *DstRC = nullptr; 5508 if (MCID.getNumDefs() > 0) { 5509 DstRC = getRegClass(MCID, 0, &RI, MF); 5510 VTs.push_back(*TRI.legalclasstypes_begin(*DstRC)); 5511 } 5512 for (unsigned i = 0, e = N->getNumValues(); i != e; ++i) { 5513 EVT VT = N->getValueType(i); 5514 if (VT != MVT::Other && i >= (unsigned)MCID.getNumDefs()) 5515 VTs.push_back(VT); 5516 } 5517 if (Load) 5518 BeforeOps.push_back(SDValue(Load, 0)); 5519 BeforeOps.insert(BeforeOps.end(), AfterOps.begin(), AfterOps.end()); 5520 // Change CMP32ri r, 0 back to TEST32rr r, r, etc. 5521 switch (Opc) { 5522 default: break; 5523 case X86::CMP64ri32: 5524 case X86::CMP64ri8: 5525 case X86::CMP32ri: 5526 case X86::CMP32ri8: 5527 case X86::CMP16ri: 5528 case X86::CMP16ri8: 5529 case X86::CMP8ri: 5530 if (isNullConstant(BeforeOps[1])) { 5531 switch (Opc) { 5532 default: llvm_unreachable("Unreachable!"); 5533 case X86::CMP64ri8: 5534 case X86::CMP64ri32: Opc = X86::TEST64rr; break; 5535 case X86::CMP32ri8: 5536 case X86::CMP32ri: Opc = X86::TEST32rr; break; 5537 case X86::CMP16ri8: 5538 case X86::CMP16ri: Opc = X86::TEST16rr; break; 5539 case X86::CMP8ri: Opc = X86::TEST8rr; break; 5540 } 5541 BeforeOps[1] = BeforeOps[0]; 5542 } 5543 } 5544 SDNode *NewNode= DAG.getMachineNode(Opc, dl, VTs, BeforeOps); 5545 NewNodes.push_back(NewNode); 5546 5547 // Emit the store instruction. 5548 if (FoldedStore) { 5549 AddrOps.pop_back(); 5550 AddrOps.push_back(SDValue(NewNode, 0)); 5551 AddrOps.push_back(Chain); 5552 std::pair<MachineInstr::mmo_iterator, 5553 MachineInstr::mmo_iterator> MMOs = 5554 MF.extractStoreMemRefs(cast<MachineSDNode>(N)->memoperands_begin(), 5555 cast<MachineSDNode>(N)->memoperands_end()); 5556 if (!(*MMOs.first) && 5557 RC == &X86::VR128RegClass && 5558 Subtarget.isUnalignedMem16Slow()) 5559 // Do not introduce a slow unaligned store. 5560 return false; 5561 // FIXME: If a VR128 can have size 32, we should be checking if a 32-byte 5562 // memory access is slow above. 5563 unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16); 5564 bool isAligned = (*MMOs.first) && 5565 (*MMOs.first)->getAlignment() >= Alignment; 5566 SDNode *Store = 5567 DAG.getMachineNode(getStoreRegOpcode(0, DstRC, isAligned, Subtarget), 5568 dl, MVT::Other, AddrOps); 5569 NewNodes.push_back(Store); 5570 5571 // Preserve memory reference information. 5572 cast<MachineSDNode>(Store)->setMemRefs(MMOs.first, MMOs.second); 5573 } 5574 5575 return true; 5576 } 5577 5578 unsigned X86InstrInfo::getOpcodeAfterMemoryUnfold(unsigned Opc, 5579 bool UnfoldLoad, bool UnfoldStore, 5580 unsigned *LoadRegIndex) const { 5581 const X86MemoryFoldTableEntry *I = lookupUnfoldTable(Opc); 5582 if (I == nullptr) 5583 return 0; 5584 bool FoldedLoad = I->Flags & TB_FOLDED_LOAD; 5585 bool FoldedStore = I->Flags & TB_FOLDED_STORE; 5586 if (UnfoldLoad && !FoldedLoad) 5587 return 0; 5588 if (UnfoldStore && !FoldedStore) 5589 return 0; 5590 if (LoadRegIndex) 5591 *LoadRegIndex = I->Flags & TB_INDEX_MASK; 5592 return I->DstOp; 5593 } 5594 5595 bool 5596 X86InstrInfo::areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2, 5597 int64_t &Offset1, int64_t &Offset2) const { 5598 if (!Load1->isMachineOpcode() || !Load2->isMachineOpcode()) 5599 return false; 5600 unsigned Opc1 = Load1->getMachineOpcode(); 5601 unsigned Opc2 = Load2->getMachineOpcode(); 5602 switch (Opc1) { 5603 default: return false; 5604 case X86::MOV8rm: 5605 case X86::MOV16rm: 5606 case X86::MOV32rm: 5607 case X86::MOV64rm: 5608 case X86::LD_Fp32m: 5609 case X86::LD_Fp64m: 5610 case X86::LD_Fp80m: 5611 case X86::MOVSSrm: 5612 case X86::MOVSDrm: 5613 case X86::MMX_MOVD64rm: 5614 case X86::MMX_MOVQ64rm: 5615 case X86::MOVAPSrm: 5616 case X86::MOVUPSrm: 5617 case X86::MOVAPDrm: 5618 case X86::MOVUPDrm: 5619 case X86::MOVDQArm: 5620 case X86::MOVDQUrm: 5621 // AVX load instructions 5622 case X86::VMOVSSrm: 5623 case X86::VMOVSDrm: 5624 case X86::VMOVAPSrm: 5625 case X86::VMOVUPSrm: 5626 case X86::VMOVAPDrm: 5627 case X86::VMOVUPDrm: 5628 case X86::VMOVDQArm: 5629 case X86::VMOVDQUrm: 5630 case X86::VMOVAPSYrm: 5631 case X86::VMOVUPSYrm: 5632 case X86::VMOVAPDYrm: 5633 case X86::VMOVUPDYrm: 5634 case X86::VMOVDQAYrm: 5635 case X86::VMOVDQUYrm: 5636 // AVX512 load instructions 5637 case X86::VMOVSSZrm: 5638 case X86::VMOVSDZrm: 5639 case X86::VMOVAPSZ128rm: 5640 case X86::VMOVUPSZ128rm: 5641 case X86::VMOVAPSZ128rm_NOVLX: 5642 case X86::VMOVUPSZ128rm_NOVLX: 5643 case X86::VMOVAPDZ128rm: 5644 case X86::VMOVUPDZ128rm: 5645 case X86::VMOVDQU8Z128rm: 5646 case X86::VMOVDQU16Z128rm: 5647 case X86::VMOVDQA32Z128rm: 5648 case X86::VMOVDQU32Z128rm: 5649 case X86::VMOVDQA64Z128rm: 5650 case X86::VMOVDQU64Z128rm: 5651 case X86::VMOVAPSZ256rm: 5652 case X86::VMOVUPSZ256rm: 5653 case X86::VMOVAPSZ256rm_NOVLX: 5654 case X86::VMOVUPSZ256rm_NOVLX: 5655 case X86::VMOVAPDZ256rm: 5656 case X86::VMOVUPDZ256rm: 5657 case X86::VMOVDQU8Z256rm: 5658 case X86::VMOVDQU16Z256rm: 5659 case X86::VMOVDQA32Z256rm: 5660 case X86::VMOVDQU32Z256rm: 5661 case X86::VMOVDQA64Z256rm: 5662 case X86::VMOVDQU64Z256rm: 5663 case X86::VMOVAPSZrm: 5664 case X86::VMOVUPSZrm: 5665 case X86::VMOVAPDZrm: 5666 case X86::VMOVUPDZrm: 5667 case X86::VMOVDQU8Zrm: 5668 case X86::VMOVDQU16Zrm: 5669 case X86::VMOVDQA32Zrm: 5670 case X86::VMOVDQU32Zrm: 5671 case X86::VMOVDQA64Zrm: 5672 case X86::VMOVDQU64Zrm: 5673 case X86::KMOVBkm: 5674 case X86::KMOVWkm: 5675 case X86::KMOVDkm: 5676 case X86::KMOVQkm: 5677 break; 5678 } 5679 switch (Opc2) { 5680 default: return false; 5681 case X86::MOV8rm: 5682 case X86::MOV16rm: 5683 case X86::MOV32rm: 5684 case X86::MOV64rm: 5685 case X86::LD_Fp32m: 5686 case X86::LD_Fp64m: 5687 case X86::LD_Fp80m: 5688 case X86::MOVSSrm: 5689 case X86::MOVSDrm: 5690 case X86::MMX_MOVD64rm: 5691 case X86::MMX_MOVQ64rm: 5692 case X86::MOVAPSrm: 5693 case X86::MOVUPSrm: 5694 case X86::MOVAPDrm: 5695 case X86::MOVUPDrm: 5696 case X86::MOVDQArm: 5697 case X86::MOVDQUrm: 5698 // AVX load instructions 5699 case X86::VMOVSSrm: 5700 case X86::VMOVSDrm: 5701 case X86::VMOVAPSrm: 5702 case X86::VMOVUPSrm: 5703 case X86::VMOVAPDrm: 5704 case X86::VMOVUPDrm: 5705 case X86::VMOVDQArm: 5706 case X86::VMOVDQUrm: 5707 case X86::VMOVAPSYrm: 5708 case X86::VMOVUPSYrm: 5709 case X86::VMOVAPDYrm: 5710 case X86::VMOVUPDYrm: 5711 case X86::VMOVDQAYrm: 5712 case X86::VMOVDQUYrm: 5713 // AVX512 load instructions 5714 case X86::VMOVSSZrm: 5715 case X86::VMOVSDZrm: 5716 case X86::VMOVAPSZ128rm: 5717 case X86::VMOVUPSZ128rm: 5718 case X86::VMOVAPSZ128rm_NOVLX: 5719 case X86::VMOVUPSZ128rm_NOVLX: 5720 case X86::VMOVAPDZ128rm: 5721 case X86::VMOVUPDZ128rm: 5722 case X86::VMOVDQU8Z128rm: 5723 case X86::VMOVDQU16Z128rm: 5724 case X86::VMOVDQA32Z128rm: 5725 case X86::VMOVDQU32Z128rm: 5726 case X86::VMOVDQA64Z128rm: 5727 case X86::VMOVDQU64Z128rm: 5728 case X86::VMOVAPSZ256rm: 5729 case X86::VMOVUPSZ256rm: 5730 case X86::VMOVAPSZ256rm_NOVLX: 5731 case X86::VMOVUPSZ256rm_NOVLX: 5732 case X86::VMOVAPDZ256rm: 5733 case X86::VMOVUPDZ256rm: 5734 case X86::VMOVDQU8Z256rm: 5735 case X86::VMOVDQU16Z256rm: 5736 case X86::VMOVDQA32Z256rm: 5737 case X86::VMOVDQU32Z256rm: 5738 case X86::VMOVDQA64Z256rm: 5739 case X86::VMOVDQU64Z256rm: 5740 case X86::VMOVAPSZrm: 5741 case X86::VMOVUPSZrm: 5742 case X86::VMOVAPDZrm: 5743 case X86::VMOVUPDZrm: 5744 case X86::VMOVDQU8Zrm: 5745 case X86::VMOVDQU16Zrm: 5746 case X86::VMOVDQA32Zrm: 5747 case X86::VMOVDQU32Zrm: 5748 case X86::VMOVDQA64Zrm: 5749 case X86::VMOVDQU64Zrm: 5750 case X86::KMOVBkm: 5751 case X86::KMOVWkm: 5752 case X86::KMOVDkm: 5753 case X86::KMOVQkm: 5754 break; 5755 } 5756 5757 // Lambda to check if both the loads have the same value for an operand index. 5758 auto HasSameOp = [&](int I) { 5759 return Load1->getOperand(I) == Load2->getOperand(I); 5760 }; 5761 5762 // All operands except the displacement should match. 5763 if (!HasSameOp(X86::AddrBaseReg) || !HasSameOp(X86::AddrScaleAmt) || 5764 !HasSameOp(X86::AddrIndexReg) || !HasSameOp(X86::AddrSegmentReg)) 5765 return false; 5766 5767 // Chain Operand must be the same. 5768 if (!HasSameOp(5)) 5769 return false; 5770 5771 // Now let's examine if the displacements are constants. 5772 auto Disp1 = dyn_cast<ConstantSDNode>(Load1->getOperand(X86::AddrDisp)); 5773 auto Disp2 = dyn_cast<ConstantSDNode>(Load2->getOperand(X86::AddrDisp)); 5774 if (!Disp1 || !Disp2) 5775 return false; 5776 5777 Offset1 = Disp1->getSExtValue(); 5778 Offset2 = Disp2->getSExtValue(); 5779 return true; 5780 } 5781 5782 bool X86InstrInfo::shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2, 5783 int64_t Offset1, int64_t Offset2, 5784 unsigned NumLoads) const { 5785 assert(Offset2 > Offset1); 5786 if ((Offset2 - Offset1) / 8 > 64) 5787 return false; 5788 5789 unsigned Opc1 = Load1->getMachineOpcode(); 5790 unsigned Opc2 = Load2->getMachineOpcode(); 5791 if (Opc1 != Opc2) 5792 return false; // FIXME: overly conservative? 5793 5794 switch (Opc1) { 5795 default: break; 5796 case X86::LD_Fp32m: 5797 case X86::LD_Fp64m: 5798 case X86::LD_Fp80m: 5799 case X86::MMX_MOVD64rm: 5800 case X86::MMX_MOVQ64rm: 5801 return false; 5802 } 5803 5804 EVT VT = Load1->getValueType(0); 5805 switch (VT.getSimpleVT().SimpleTy) { 5806 default: 5807 // XMM registers. In 64-bit mode we can be a bit more aggressive since we 5808 // have 16 of them to play with. 5809 if (Subtarget.is64Bit()) { 5810 if (NumLoads >= 3) 5811 return false; 5812 } else if (NumLoads) { 5813 return false; 5814 } 5815 break; 5816 case MVT::i8: 5817 case MVT::i16: 5818 case MVT::i32: 5819 case MVT::i64: 5820 case MVT::f32: 5821 case MVT::f64: 5822 if (NumLoads) 5823 return false; 5824 break; 5825 } 5826 5827 return true; 5828 } 5829 5830 bool X86InstrInfo:: 5831 reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const { 5832 assert(Cond.size() == 1 && "Invalid X86 branch condition!"); 5833 X86::CondCode CC = static_cast<X86::CondCode>(Cond[0].getImm()); 5834 Cond[0].setImm(GetOppositeBranchCondition(CC)); 5835 return false; 5836 } 5837 5838 bool X86InstrInfo:: 5839 isSafeToMoveRegClassDefs(const TargetRegisterClass *RC) const { 5840 // FIXME: Return false for x87 stack register classes for now. We can't 5841 // allow any loads of these registers before FpGet_ST0_80. 5842 return !(RC == &X86::CCRRegClass || RC == &X86::DFCCRRegClass || 5843 RC == &X86::RFP32RegClass || RC == &X86::RFP64RegClass || 5844 RC == &X86::RFP80RegClass); 5845 } 5846 5847 /// Return a virtual register initialized with the 5848 /// the global base register value. Output instructions required to 5849 /// initialize the register in the function entry block, if necessary. 5850 /// 5851 /// TODO: Eliminate this and move the code to X86MachineFunctionInfo. 5852 /// 5853 unsigned X86InstrInfo::getGlobalBaseReg(MachineFunction *MF) const { 5854 assert(!Subtarget.is64Bit() && 5855 "X86-64 PIC uses RIP relative addressing"); 5856 5857 X86MachineFunctionInfo *X86FI = MF->getInfo<X86MachineFunctionInfo>(); 5858 unsigned GlobalBaseReg = X86FI->getGlobalBaseReg(); 5859 if (GlobalBaseReg != 0) 5860 return GlobalBaseReg; 5861 5862 // Create the register. The code to initialize it is inserted 5863 // later, by the CGBR pass (below). 5864 MachineRegisterInfo &RegInfo = MF->getRegInfo(); 5865 GlobalBaseReg = RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass); 5866 X86FI->setGlobalBaseReg(GlobalBaseReg); 5867 return GlobalBaseReg; 5868 } 5869 5870 // These are the replaceable SSE instructions. Some of these have Int variants 5871 // that we don't include here. We don't want to replace instructions selected 5872 // by intrinsics. 5873 static const uint16_t ReplaceableInstrs[][3] = { 5874 //PackedSingle PackedDouble PackedInt 5875 { X86::MOVAPSmr, X86::MOVAPDmr, X86::MOVDQAmr }, 5876 { X86::MOVAPSrm, X86::MOVAPDrm, X86::MOVDQArm }, 5877 { X86::MOVAPSrr, X86::MOVAPDrr, X86::MOVDQArr }, 5878 { X86::MOVUPSmr, X86::MOVUPDmr, X86::MOVDQUmr }, 5879 { X86::MOVUPSrm, X86::MOVUPDrm, X86::MOVDQUrm }, 5880 { X86::MOVLPSmr, X86::MOVLPDmr, X86::MOVPQI2QImr }, 5881 { X86::MOVSDmr, X86::MOVSDmr, X86::MOVPQI2QImr }, 5882 { X86::MOVSSmr, X86::MOVSSmr, X86::MOVPDI2DImr }, 5883 { X86::MOVSDrm, X86::MOVSDrm, X86::MOVQI2PQIrm }, 5884 { X86::MOVSSrm, X86::MOVSSrm, X86::MOVDI2PDIrm }, 5885 { X86::MOVNTPSmr, X86::MOVNTPDmr, X86::MOVNTDQmr }, 5886 { X86::ANDNPSrm, X86::ANDNPDrm, X86::PANDNrm }, 5887 { X86::ANDNPSrr, X86::ANDNPDrr, X86::PANDNrr }, 5888 { X86::ANDPSrm, X86::ANDPDrm, X86::PANDrm }, 5889 { X86::ANDPSrr, X86::ANDPDrr, X86::PANDrr }, 5890 { X86::ORPSrm, X86::ORPDrm, X86::PORrm }, 5891 { X86::ORPSrr, X86::ORPDrr, X86::PORrr }, 5892 { X86::XORPSrm, X86::XORPDrm, X86::PXORrm }, 5893 { X86::XORPSrr, X86::XORPDrr, X86::PXORrr }, 5894 { X86::UNPCKLPDrm, X86::UNPCKLPDrm, X86::PUNPCKLQDQrm }, 5895 { X86::MOVLHPSrr, X86::UNPCKLPDrr, X86::PUNPCKLQDQrr }, 5896 { X86::UNPCKHPDrm, X86::UNPCKHPDrm, X86::PUNPCKHQDQrm }, 5897 { X86::UNPCKHPDrr, X86::UNPCKHPDrr, X86::PUNPCKHQDQrr }, 5898 { X86::UNPCKLPSrm, X86::UNPCKLPSrm, X86::PUNPCKLDQrm }, 5899 { X86::UNPCKLPSrr, X86::UNPCKLPSrr, X86::PUNPCKLDQrr }, 5900 { X86::UNPCKHPSrm, X86::UNPCKHPSrm, X86::PUNPCKHDQrm }, 5901 { X86::UNPCKHPSrr, X86::UNPCKHPSrr, X86::PUNPCKHDQrr }, 5902 { X86::EXTRACTPSmr, X86::EXTRACTPSmr, X86::PEXTRDmr }, 5903 { X86::EXTRACTPSrr, X86::EXTRACTPSrr, X86::PEXTRDrr }, 5904 // AVX 128-bit support 5905 { X86::VMOVAPSmr, X86::VMOVAPDmr, X86::VMOVDQAmr }, 5906 { X86::VMOVAPSrm, X86::VMOVAPDrm, X86::VMOVDQArm }, 5907 { X86::VMOVAPSrr, X86::VMOVAPDrr, X86::VMOVDQArr }, 5908 { X86::VMOVUPSmr, X86::VMOVUPDmr, X86::VMOVDQUmr }, 5909 { X86::VMOVUPSrm, X86::VMOVUPDrm, X86::VMOVDQUrm }, 5910 { X86::VMOVLPSmr, X86::VMOVLPDmr, X86::VMOVPQI2QImr }, 5911 { X86::VMOVSDmr, X86::VMOVSDmr, X86::VMOVPQI2QImr }, 5912 { X86::VMOVSSmr, X86::VMOVSSmr, X86::VMOVPDI2DImr }, 5913 { X86::VMOVSDrm, X86::VMOVSDrm, X86::VMOVQI2PQIrm }, 5914 { X86::VMOVSSrm, X86::VMOVSSrm, X86::VMOVDI2PDIrm }, 5915 { X86::VMOVNTPSmr, X86::VMOVNTPDmr, X86::VMOVNTDQmr }, 5916 { X86::VANDNPSrm, X86::VANDNPDrm, X86::VPANDNrm }, 5917 { X86::VANDNPSrr, X86::VANDNPDrr, X86::VPANDNrr }, 5918 { X86::VANDPSrm, X86::VANDPDrm, X86::VPANDrm }, 5919 { X86::VANDPSrr, X86::VANDPDrr, X86::VPANDrr }, 5920 { X86::VORPSrm, X86::VORPDrm, X86::VPORrm }, 5921 { X86::VORPSrr, X86::VORPDrr, X86::VPORrr }, 5922 { X86::VXORPSrm, X86::VXORPDrm, X86::VPXORrm }, 5923 { X86::VXORPSrr, X86::VXORPDrr, X86::VPXORrr }, 5924 { X86::VUNPCKLPDrm, X86::VUNPCKLPDrm, X86::VPUNPCKLQDQrm }, 5925 { X86::VMOVLHPSrr, X86::VUNPCKLPDrr, X86::VPUNPCKLQDQrr }, 5926 { X86::VUNPCKHPDrm, X86::VUNPCKHPDrm, X86::VPUNPCKHQDQrm }, 5927 { X86::VUNPCKHPDrr, X86::VUNPCKHPDrr, X86::VPUNPCKHQDQrr }, 5928 { X86::VUNPCKLPSrm, X86::VUNPCKLPSrm, X86::VPUNPCKLDQrm }, 5929 { X86::VUNPCKLPSrr, X86::VUNPCKLPSrr, X86::VPUNPCKLDQrr }, 5930 { X86::VUNPCKHPSrm, X86::VUNPCKHPSrm, X86::VPUNPCKHDQrm }, 5931 { X86::VUNPCKHPSrr, X86::VUNPCKHPSrr, X86::VPUNPCKHDQrr }, 5932 { X86::VEXTRACTPSmr, X86::VEXTRACTPSmr, X86::VPEXTRDmr }, 5933 { X86::VEXTRACTPSrr, X86::VEXTRACTPSrr, X86::VPEXTRDrr }, 5934 // AVX 256-bit support 5935 { X86::VMOVAPSYmr, X86::VMOVAPDYmr, X86::VMOVDQAYmr }, 5936 { X86::VMOVAPSYrm, X86::VMOVAPDYrm, X86::VMOVDQAYrm }, 5937 { X86::VMOVAPSYrr, X86::VMOVAPDYrr, X86::VMOVDQAYrr }, 5938 { X86::VMOVUPSYmr, X86::VMOVUPDYmr, X86::VMOVDQUYmr }, 5939 { X86::VMOVUPSYrm, X86::VMOVUPDYrm, X86::VMOVDQUYrm }, 5940 { X86::VMOVNTPSYmr, X86::VMOVNTPDYmr, X86::VMOVNTDQYmr }, 5941 { X86::VPERMPSYrm, X86::VPERMPSYrm, X86::VPERMDYrm }, 5942 { X86::VPERMPSYrr, X86::VPERMPSYrr, X86::VPERMDYrr }, 5943 { X86::VPERMPDYmi, X86::VPERMPDYmi, X86::VPERMQYmi }, 5944 { X86::VPERMPDYri, X86::VPERMPDYri, X86::VPERMQYri }, 5945 // AVX512 support 5946 { X86::VMOVLPSZ128mr, X86::VMOVLPDZ128mr, X86::VMOVPQI2QIZmr }, 5947 { X86::VMOVNTPSZ128mr, X86::VMOVNTPDZ128mr, X86::VMOVNTDQZ128mr }, 5948 { X86::VMOVNTPSZ256mr, X86::VMOVNTPDZ256mr, X86::VMOVNTDQZ256mr }, 5949 { X86::VMOVNTPSZmr, X86::VMOVNTPDZmr, X86::VMOVNTDQZmr }, 5950 { X86::VMOVSDZmr, X86::VMOVSDZmr, X86::VMOVPQI2QIZmr }, 5951 { X86::VMOVSSZmr, X86::VMOVSSZmr, X86::VMOVPDI2DIZmr }, 5952 { X86::VMOVSDZrm, X86::VMOVSDZrm, X86::VMOVQI2PQIZrm }, 5953 { X86::VMOVSSZrm, X86::VMOVSSZrm, X86::VMOVDI2PDIZrm }, 5954 { X86::VBROADCASTSSZ128r, X86::VBROADCASTSSZ128r, X86::VPBROADCASTDZ128r }, 5955 { X86::VBROADCASTSSZ128m, X86::VBROADCASTSSZ128m, X86::VPBROADCASTDZ128m }, 5956 { X86::VBROADCASTSSZ256r, X86::VBROADCASTSSZ256r, X86::VPBROADCASTDZ256r }, 5957 { X86::VBROADCASTSSZ256m, X86::VBROADCASTSSZ256m, X86::VPBROADCASTDZ256m }, 5958 { X86::VBROADCASTSSZr, X86::VBROADCASTSSZr, X86::VPBROADCASTDZr }, 5959 { X86::VBROADCASTSSZm, X86::VBROADCASTSSZm, X86::VPBROADCASTDZm }, 5960 { X86::VBROADCASTSDZ256r, X86::VBROADCASTSDZ256r, X86::VPBROADCASTQZ256r }, 5961 { X86::VBROADCASTSDZ256m, X86::VBROADCASTSDZ256m, X86::VPBROADCASTQZ256m }, 5962 { X86::VBROADCASTSDZr, X86::VBROADCASTSDZr, X86::VPBROADCASTQZr }, 5963 { X86::VBROADCASTSDZm, X86::VBROADCASTSDZm, X86::VPBROADCASTQZm }, 5964 { X86::VINSERTF32x4Zrr, X86::VINSERTF32x4Zrr, X86::VINSERTI32x4Zrr }, 5965 { X86::VINSERTF32x4Zrm, X86::VINSERTF32x4Zrm, X86::VINSERTI32x4Zrm }, 5966 { X86::VINSERTF32x8Zrr, X86::VINSERTF32x8Zrr, X86::VINSERTI32x8Zrr }, 5967 { X86::VINSERTF32x8Zrm, X86::VINSERTF32x8Zrm, X86::VINSERTI32x8Zrm }, 5968 { X86::VINSERTF64x2Zrr, X86::VINSERTF64x2Zrr, X86::VINSERTI64x2Zrr }, 5969 { X86::VINSERTF64x2Zrm, X86::VINSERTF64x2Zrm, X86::VINSERTI64x2Zrm }, 5970 { X86::VINSERTF64x4Zrr, X86::VINSERTF64x4Zrr, X86::VINSERTI64x4Zrr }, 5971 { X86::VINSERTF64x4Zrm, X86::VINSERTF64x4Zrm, X86::VINSERTI64x4Zrm }, 5972 { X86::VINSERTF32x4Z256rr,X86::VINSERTF32x4Z256rr,X86::VINSERTI32x4Z256rr }, 5973 { X86::VINSERTF32x4Z256rm,X86::VINSERTF32x4Z256rm,X86::VINSERTI32x4Z256rm }, 5974 { X86::VINSERTF64x2Z256rr,X86::VINSERTF64x2Z256rr,X86::VINSERTI64x2Z256rr }, 5975 { X86::VINSERTF64x2Z256rm,X86::VINSERTF64x2Z256rm,X86::VINSERTI64x2Z256rm }, 5976 { X86::VEXTRACTF32x4Zrr, X86::VEXTRACTF32x4Zrr, X86::VEXTRACTI32x4Zrr }, 5977 { X86::VEXTRACTF32x4Zmr, X86::VEXTRACTF32x4Zmr, X86::VEXTRACTI32x4Zmr }, 5978 { X86::VEXTRACTF32x8Zrr, X86::VEXTRACTF32x8Zrr, X86::VEXTRACTI32x8Zrr }, 5979 { X86::VEXTRACTF32x8Zmr, X86::VEXTRACTF32x8Zmr, X86::VEXTRACTI32x8Zmr }, 5980 { X86::VEXTRACTF64x2Zrr, X86::VEXTRACTF64x2Zrr, X86::VEXTRACTI64x2Zrr }, 5981 { X86::VEXTRACTF64x2Zmr, X86::VEXTRACTF64x2Zmr, X86::VEXTRACTI64x2Zmr }, 5982 { X86::VEXTRACTF64x4Zrr, X86::VEXTRACTF64x4Zrr, X86::VEXTRACTI64x4Zrr }, 5983 { X86::VEXTRACTF64x4Zmr, X86::VEXTRACTF64x4Zmr, X86::VEXTRACTI64x4Zmr }, 5984 { X86::VEXTRACTF32x4Z256rr,X86::VEXTRACTF32x4Z256rr,X86::VEXTRACTI32x4Z256rr }, 5985 { X86::VEXTRACTF32x4Z256mr,X86::VEXTRACTF32x4Z256mr,X86::VEXTRACTI32x4Z256mr }, 5986 { X86::VEXTRACTF64x2Z256rr,X86::VEXTRACTF64x2Z256rr,X86::VEXTRACTI64x2Z256rr }, 5987 { X86::VEXTRACTF64x2Z256mr,X86::VEXTRACTF64x2Z256mr,X86::VEXTRACTI64x2Z256mr }, 5988 { X86::VPERMILPSmi, X86::VPERMILPSmi, X86::VPSHUFDmi }, 5989 { X86::VPERMILPSri, X86::VPERMILPSri, X86::VPSHUFDri }, 5990 { X86::VPERMILPSZ128mi, X86::VPERMILPSZ128mi, X86::VPSHUFDZ128mi }, 5991 { X86::VPERMILPSZ128ri, X86::VPERMILPSZ128ri, X86::VPSHUFDZ128ri }, 5992 { X86::VPERMILPSZ256mi, X86::VPERMILPSZ256mi, X86::VPSHUFDZ256mi }, 5993 { X86::VPERMILPSZ256ri, X86::VPERMILPSZ256ri, X86::VPSHUFDZ256ri }, 5994 { X86::VPERMILPSZmi, X86::VPERMILPSZmi, X86::VPSHUFDZmi }, 5995 { X86::VPERMILPSZri, X86::VPERMILPSZri, X86::VPSHUFDZri }, 5996 { X86::VPERMPSZ256rm, X86::VPERMPSZ256rm, X86::VPERMDZ256rm }, 5997 { X86::VPERMPSZ256rr, X86::VPERMPSZ256rr, X86::VPERMDZ256rr }, 5998 { X86::VPERMPDZ256mi, X86::VPERMPDZ256mi, X86::VPERMQZ256mi }, 5999 { X86::VPERMPDZ256ri, X86::VPERMPDZ256ri, X86::VPERMQZ256ri }, 6000 { X86::VPERMPDZ256rm, X86::VPERMPDZ256rm, X86::VPERMQZ256rm }, 6001 { X86::VPERMPDZ256rr, X86::VPERMPDZ256rr, X86::VPERMQZ256rr }, 6002 { X86::VPERMPSZrm, X86::VPERMPSZrm, X86::VPERMDZrm }, 6003 { X86::VPERMPSZrr, X86::VPERMPSZrr, X86::VPERMDZrr }, 6004 { X86::VPERMPDZmi, X86::VPERMPDZmi, X86::VPERMQZmi }, 6005 { X86::VPERMPDZri, X86::VPERMPDZri, X86::VPERMQZri }, 6006 { X86::VPERMPDZrm, X86::VPERMPDZrm, X86::VPERMQZrm }, 6007 { X86::VPERMPDZrr, X86::VPERMPDZrr, X86::VPERMQZrr }, 6008 { X86::VUNPCKLPDZ256rm, X86::VUNPCKLPDZ256rm, X86::VPUNPCKLQDQZ256rm }, 6009 { X86::VUNPCKLPDZ256rr, X86::VUNPCKLPDZ256rr, X86::VPUNPCKLQDQZ256rr }, 6010 { X86::VUNPCKHPDZ256rm, X86::VUNPCKHPDZ256rm, X86::VPUNPCKHQDQZ256rm }, 6011 { X86::VUNPCKHPDZ256rr, X86::VUNPCKHPDZ256rr, X86::VPUNPCKHQDQZ256rr }, 6012 { X86::VUNPCKLPSZ256rm, X86::VUNPCKLPSZ256rm, X86::VPUNPCKLDQZ256rm }, 6013 { X86::VUNPCKLPSZ256rr, X86::VUNPCKLPSZ256rr, X86::VPUNPCKLDQZ256rr }, 6014 { X86::VUNPCKHPSZ256rm, X86::VUNPCKHPSZ256rm, X86::VPUNPCKHDQZ256rm }, 6015 { X86::VUNPCKHPSZ256rr, X86::VUNPCKHPSZ256rr, X86::VPUNPCKHDQZ256rr }, 6016 { X86::VUNPCKLPDZ128rm, X86::VUNPCKLPDZ128rm, X86::VPUNPCKLQDQZ128rm }, 6017 { X86::VMOVLHPSZrr, X86::VUNPCKLPDZ128rr, X86::VPUNPCKLQDQZ128rr }, 6018 { X86::VUNPCKHPDZ128rm, X86::VUNPCKHPDZ128rm, X86::VPUNPCKHQDQZ128rm }, 6019 { X86::VUNPCKHPDZ128rr, X86::VUNPCKHPDZ128rr, X86::VPUNPCKHQDQZ128rr }, 6020 { X86::VUNPCKLPSZ128rm, X86::VUNPCKLPSZ128rm, X86::VPUNPCKLDQZ128rm }, 6021 { X86::VUNPCKLPSZ128rr, X86::VUNPCKLPSZ128rr, X86::VPUNPCKLDQZ128rr }, 6022 { X86::VUNPCKHPSZ128rm, X86::VUNPCKHPSZ128rm, X86::VPUNPCKHDQZ128rm }, 6023 { X86::VUNPCKHPSZ128rr, X86::VUNPCKHPSZ128rr, X86::VPUNPCKHDQZ128rr }, 6024 { X86::VUNPCKLPDZrm, X86::VUNPCKLPDZrm, X86::VPUNPCKLQDQZrm }, 6025 { X86::VUNPCKLPDZrr, X86::VUNPCKLPDZrr, X86::VPUNPCKLQDQZrr }, 6026 { X86::VUNPCKHPDZrm, X86::VUNPCKHPDZrm, X86::VPUNPCKHQDQZrm }, 6027 { X86::VUNPCKHPDZrr, X86::VUNPCKHPDZrr, X86::VPUNPCKHQDQZrr }, 6028 { X86::VUNPCKLPSZrm, X86::VUNPCKLPSZrm, X86::VPUNPCKLDQZrm }, 6029 { X86::VUNPCKLPSZrr, X86::VUNPCKLPSZrr, X86::VPUNPCKLDQZrr }, 6030 { X86::VUNPCKHPSZrm, X86::VUNPCKHPSZrm, X86::VPUNPCKHDQZrm }, 6031 { X86::VUNPCKHPSZrr, X86::VUNPCKHPSZrr, X86::VPUNPCKHDQZrr }, 6032 { X86::VEXTRACTPSZmr, X86::VEXTRACTPSZmr, X86::VPEXTRDZmr }, 6033 { X86::VEXTRACTPSZrr, X86::VEXTRACTPSZrr, X86::VPEXTRDZrr }, 6034 }; 6035 6036 static const uint16_t ReplaceableInstrsAVX2[][3] = { 6037 //PackedSingle PackedDouble PackedInt 6038 { X86::VANDNPSYrm, X86::VANDNPDYrm, X86::VPANDNYrm }, 6039 { X86::VANDNPSYrr, X86::VANDNPDYrr, X86::VPANDNYrr }, 6040 { X86::VANDPSYrm, X86::VANDPDYrm, X86::VPANDYrm }, 6041 { X86::VANDPSYrr, X86::VANDPDYrr, X86::VPANDYrr }, 6042 { X86::VORPSYrm, X86::VORPDYrm, X86::VPORYrm }, 6043 { X86::VORPSYrr, X86::VORPDYrr, X86::VPORYrr }, 6044 { X86::VXORPSYrm, X86::VXORPDYrm, X86::VPXORYrm }, 6045 { X86::VXORPSYrr, X86::VXORPDYrr, X86::VPXORYrr }, 6046 { X86::VPERM2F128rm, X86::VPERM2F128rm, X86::VPERM2I128rm }, 6047 { X86::VPERM2F128rr, X86::VPERM2F128rr, X86::VPERM2I128rr }, 6048 { X86::VBROADCASTSSrm, X86::VBROADCASTSSrm, X86::VPBROADCASTDrm}, 6049 { X86::VBROADCASTSSrr, X86::VBROADCASTSSrr, X86::VPBROADCASTDrr}, 6050 { X86::VBROADCASTSSYrr, X86::VBROADCASTSSYrr, X86::VPBROADCASTDYrr}, 6051 { X86::VBROADCASTSSYrm, X86::VBROADCASTSSYrm, X86::VPBROADCASTDYrm}, 6052 { X86::VBROADCASTSDYrr, X86::VBROADCASTSDYrr, X86::VPBROADCASTQYrr}, 6053 { X86::VBROADCASTSDYrm, X86::VBROADCASTSDYrm, X86::VPBROADCASTQYrm}, 6054 { X86::VBROADCASTF128, X86::VBROADCASTF128, X86::VBROADCASTI128 }, 6055 { X86::VBLENDPSYrri, X86::VBLENDPSYrri, X86::VPBLENDDYrri }, 6056 { X86::VBLENDPSYrmi, X86::VBLENDPSYrmi, X86::VPBLENDDYrmi }, 6057 { X86::VPERMILPSYmi, X86::VPERMILPSYmi, X86::VPSHUFDYmi }, 6058 { X86::VPERMILPSYri, X86::VPERMILPSYri, X86::VPSHUFDYri }, 6059 { X86::VUNPCKLPDYrm, X86::VUNPCKLPDYrm, X86::VPUNPCKLQDQYrm }, 6060 { X86::VUNPCKLPDYrr, X86::VUNPCKLPDYrr, X86::VPUNPCKLQDQYrr }, 6061 { X86::VUNPCKHPDYrm, X86::VUNPCKHPDYrm, X86::VPUNPCKHQDQYrm }, 6062 { X86::VUNPCKHPDYrr, X86::VUNPCKHPDYrr, X86::VPUNPCKHQDQYrr }, 6063 { X86::VUNPCKLPSYrm, X86::VUNPCKLPSYrm, X86::VPUNPCKLDQYrm }, 6064 { X86::VUNPCKLPSYrr, X86::VUNPCKLPSYrr, X86::VPUNPCKLDQYrr }, 6065 { X86::VUNPCKHPSYrm, X86::VUNPCKHPSYrm, X86::VPUNPCKHDQYrm }, 6066 { X86::VUNPCKHPSYrr, X86::VUNPCKHPSYrr, X86::VPUNPCKHDQYrr }, 6067 }; 6068 6069 static const uint16_t ReplaceableInstrsAVX2InsertExtract[][3] = { 6070 //PackedSingle PackedDouble PackedInt 6071 { X86::VEXTRACTF128mr, X86::VEXTRACTF128mr, X86::VEXTRACTI128mr }, 6072 { X86::VEXTRACTF128rr, X86::VEXTRACTF128rr, X86::VEXTRACTI128rr }, 6073 { X86::VINSERTF128rm, X86::VINSERTF128rm, X86::VINSERTI128rm }, 6074 { X86::VINSERTF128rr, X86::VINSERTF128rr, X86::VINSERTI128rr }, 6075 }; 6076 6077 static const uint16_t ReplaceableInstrsAVX512[][4] = { 6078 // Two integer columns for 64-bit and 32-bit elements. 6079 //PackedSingle PackedDouble PackedInt PackedInt 6080 { X86::VMOVAPSZ128mr, X86::VMOVAPDZ128mr, X86::VMOVDQA64Z128mr, X86::VMOVDQA32Z128mr }, 6081 { X86::VMOVAPSZ128rm, X86::VMOVAPDZ128rm, X86::VMOVDQA64Z128rm, X86::VMOVDQA32Z128rm }, 6082 { X86::VMOVAPSZ128rr, X86::VMOVAPDZ128rr, X86::VMOVDQA64Z128rr, X86::VMOVDQA32Z128rr }, 6083 { X86::VMOVUPSZ128mr, X86::VMOVUPDZ128mr, X86::VMOVDQU64Z128mr, X86::VMOVDQU32Z128mr }, 6084 { X86::VMOVUPSZ128rm, X86::VMOVUPDZ128rm, X86::VMOVDQU64Z128rm, X86::VMOVDQU32Z128rm }, 6085 { X86::VMOVAPSZ256mr, X86::VMOVAPDZ256mr, X86::VMOVDQA64Z256mr, X86::VMOVDQA32Z256mr }, 6086 { X86::VMOVAPSZ256rm, X86::VMOVAPDZ256rm, X86::VMOVDQA64Z256rm, X86::VMOVDQA32Z256rm }, 6087 { X86::VMOVAPSZ256rr, X86::VMOVAPDZ256rr, X86::VMOVDQA64Z256rr, X86::VMOVDQA32Z256rr }, 6088 { X86::VMOVUPSZ256mr, X86::VMOVUPDZ256mr, X86::VMOVDQU64Z256mr, X86::VMOVDQU32Z256mr }, 6089 { X86::VMOVUPSZ256rm, X86::VMOVUPDZ256rm, X86::VMOVDQU64Z256rm, X86::VMOVDQU32Z256rm }, 6090 { X86::VMOVAPSZmr, X86::VMOVAPDZmr, X86::VMOVDQA64Zmr, X86::VMOVDQA32Zmr }, 6091 { X86::VMOVAPSZrm, X86::VMOVAPDZrm, X86::VMOVDQA64Zrm, X86::VMOVDQA32Zrm }, 6092 { X86::VMOVAPSZrr, X86::VMOVAPDZrr, X86::VMOVDQA64Zrr, X86::VMOVDQA32Zrr }, 6093 { X86::VMOVUPSZmr, X86::VMOVUPDZmr, X86::VMOVDQU64Zmr, X86::VMOVDQU32Zmr }, 6094 { X86::VMOVUPSZrm, X86::VMOVUPDZrm, X86::VMOVDQU64Zrm, X86::VMOVDQU32Zrm }, 6095 }; 6096 6097 static const uint16_t ReplaceableInstrsAVX512DQ[][4] = { 6098 // Two integer columns for 64-bit and 32-bit elements. 6099 //PackedSingle PackedDouble PackedInt PackedInt 6100 { X86::VANDNPSZ128rm, X86::VANDNPDZ128rm, X86::VPANDNQZ128rm, X86::VPANDNDZ128rm }, 6101 { X86::VANDNPSZ128rr, X86::VANDNPDZ128rr, X86::VPANDNQZ128rr, X86::VPANDNDZ128rr }, 6102 { X86::VANDPSZ128rm, X86::VANDPDZ128rm, X86::VPANDQZ128rm, X86::VPANDDZ128rm }, 6103 { X86::VANDPSZ128rr, X86::VANDPDZ128rr, X86::VPANDQZ128rr, X86::VPANDDZ128rr }, 6104 { X86::VORPSZ128rm, X86::VORPDZ128rm, X86::VPORQZ128rm, X86::VPORDZ128rm }, 6105 { X86::VORPSZ128rr, X86::VORPDZ128rr, X86::VPORQZ128rr, X86::VPORDZ128rr }, 6106 { X86::VXORPSZ128rm, X86::VXORPDZ128rm, X86::VPXORQZ128rm, X86::VPXORDZ128rm }, 6107 { X86::VXORPSZ128rr, X86::VXORPDZ128rr, X86::VPXORQZ128rr, X86::VPXORDZ128rr }, 6108 { X86::VANDNPSZ256rm, X86::VANDNPDZ256rm, X86::VPANDNQZ256rm, X86::VPANDNDZ256rm }, 6109 { X86::VANDNPSZ256rr, X86::VANDNPDZ256rr, X86::VPANDNQZ256rr, X86::VPANDNDZ256rr }, 6110 { X86::VANDPSZ256rm, X86::VANDPDZ256rm, X86::VPANDQZ256rm, X86::VPANDDZ256rm }, 6111 { X86::VANDPSZ256rr, X86::VANDPDZ256rr, X86::VPANDQZ256rr, X86::VPANDDZ256rr }, 6112 { X86::VORPSZ256rm, X86::VORPDZ256rm, X86::VPORQZ256rm, X86::VPORDZ256rm }, 6113 { X86::VORPSZ256rr, X86::VORPDZ256rr, X86::VPORQZ256rr, X86::VPORDZ256rr }, 6114 { X86::VXORPSZ256rm, X86::VXORPDZ256rm, X86::VPXORQZ256rm, X86::VPXORDZ256rm }, 6115 { X86::VXORPSZ256rr, X86::VXORPDZ256rr, X86::VPXORQZ256rr, X86::VPXORDZ256rr }, 6116 { X86::VANDNPSZrm, X86::VANDNPDZrm, X86::VPANDNQZrm, X86::VPANDNDZrm }, 6117 { X86::VANDNPSZrr, X86::VANDNPDZrr, X86::VPANDNQZrr, X86::VPANDNDZrr }, 6118 { X86::VANDPSZrm, X86::VANDPDZrm, X86::VPANDQZrm, X86::VPANDDZrm }, 6119 { X86::VANDPSZrr, X86::VANDPDZrr, X86::VPANDQZrr, X86::VPANDDZrr }, 6120 { X86::VORPSZrm, X86::VORPDZrm, X86::VPORQZrm, X86::VPORDZrm }, 6121 { X86::VORPSZrr, X86::VORPDZrr, X86::VPORQZrr, X86::VPORDZrr }, 6122 { X86::VXORPSZrm, X86::VXORPDZrm, X86::VPXORQZrm, X86::VPXORDZrm }, 6123 { X86::VXORPSZrr, X86::VXORPDZrr, X86::VPXORQZrr, X86::VPXORDZrr }, 6124 }; 6125 6126 static const uint16_t ReplaceableInstrsAVX512DQMasked[][4] = { 6127 // Two integer columns for 64-bit and 32-bit elements. 6128 //PackedSingle PackedDouble 6129 //PackedInt PackedInt 6130 { X86::VANDNPSZ128rmk, X86::VANDNPDZ128rmk, 6131 X86::VPANDNQZ128rmk, X86::VPANDNDZ128rmk }, 6132 { X86::VANDNPSZ128rmkz, X86::VANDNPDZ128rmkz, 6133 X86::VPANDNQZ128rmkz, X86::VPANDNDZ128rmkz }, 6134 { X86::VANDNPSZ128rrk, X86::VANDNPDZ128rrk, 6135 X86::VPANDNQZ128rrk, X86::VPANDNDZ128rrk }, 6136 { X86::VANDNPSZ128rrkz, X86::VANDNPDZ128rrkz, 6137 X86::VPANDNQZ128rrkz, X86::VPANDNDZ128rrkz }, 6138 { X86::VANDPSZ128rmk, X86::VANDPDZ128rmk, 6139 X86::VPANDQZ128rmk, X86::VPANDDZ128rmk }, 6140 { X86::VANDPSZ128rmkz, X86::VANDPDZ128rmkz, 6141 X86::VPANDQZ128rmkz, X86::VPANDDZ128rmkz }, 6142 { X86::VANDPSZ128rrk, X86::VANDPDZ128rrk, 6143 X86::VPANDQZ128rrk, X86::VPANDDZ128rrk }, 6144 { X86::VANDPSZ128rrkz, X86::VANDPDZ128rrkz, 6145 X86::VPANDQZ128rrkz, X86::VPANDDZ128rrkz }, 6146 { X86::VORPSZ128rmk, X86::VORPDZ128rmk, 6147 X86::VPORQZ128rmk, X86::VPORDZ128rmk }, 6148 { X86::VORPSZ128rmkz, X86::VORPDZ128rmkz, 6149 X86::VPORQZ128rmkz, X86::VPORDZ128rmkz }, 6150 { X86::VORPSZ128rrk, X86::VORPDZ128rrk, 6151 X86::VPORQZ128rrk, X86::VPORDZ128rrk }, 6152 { X86::VORPSZ128rrkz, X86::VORPDZ128rrkz, 6153 X86::VPORQZ128rrkz, X86::VPORDZ128rrkz }, 6154 { X86::VXORPSZ128rmk, X86::VXORPDZ128rmk, 6155 X86::VPXORQZ128rmk, X86::VPXORDZ128rmk }, 6156 { X86::VXORPSZ128rmkz, X86::VXORPDZ128rmkz, 6157 X86::VPXORQZ128rmkz, X86::VPXORDZ128rmkz }, 6158 { X86::VXORPSZ128rrk, X86::VXORPDZ128rrk, 6159 X86::VPXORQZ128rrk, X86::VPXORDZ128rrk }, 6160 { X86::VXORPSZ128rrkz, X86::VXORPDZ128rrkz, 6161 X86::VPXORQZ128rrkz, X86::VPXORDZ128rrkz }, 6162 { X86::VANDNPSZ256rmk, X86::VANDNPDZ256rmk, 6163 X86::VPANDNQZ256rmk, X86::VPANDNDZ256rmk }, 6164 { X86::VANDNPSZ256rmkz, X86::VANDNPDZ256rmkz, 6165 X86::VPANDNQZ256rmkz, X86::VPANDNDZ256rmkz }, 6166 { X86::VANDNPSZ256rrk, X86::VANDNPDZ256rrk, 6167 X86::VPANDNQZ256rrk, X86::VPANDNDZ256rrk }, 6168 { X86::VANDNPSZ256rrkz, X86::VANDNPDZ256rrkz, 6169 X86::VPANDNQZ256rrkz, X86::VPANDNDZ256rrkz }, 6170 { X86::VANDPSZ256rmk, X86::VANDPDZ256rmk, 6171 X86::VPANDQZ256rmk, X86::VPANDDZ256rmk }, 6172 { X86::VANDPSZ256rmkz, X86::VANDPDZ256rmkz, 6173 X86::VPANDQZ256rmkz, X86::VPANDDZ256rmkz }, 6174 { X86::VANDPSZ256rrk, X86::VANDPDZ256rrk, 6175 X86::VPANDQZ256rrk, X86::VPANDDZ256rrk }, 6176 { X86::VANDPSZ256rrkz, X86::VANDPDZ256rrkz, 6177 X86::VPANDQZ256rrkz, X86::VPANDDZ256rrkz }, 6178 { X86::VORPSZ256rmk, X86::VORPDZ256rmk, 6179 X86::VPORQZ256rmk, X86::VPORDZ256rmk }, 6180 { X86::VORPSZ256rmkz, X86::VORPDZ256rmkz, 6181 X86::VPORQZ256rmkz, X86::VPORDZ256rmkz }, 6182 { X86::VORPSZ256rrk, X86::VORPDZ256rrk, 6183 X86::VPORQZ256rrk, X86::VPORDZ256rrk }, 6184 { X86::VORPSZ256rrkz, X86::VORPDZ256rrkz, 6185 X86::VPORQZ256rrkz, X86::VPORDZ256rrkz }, 6186 { X86::VXORPSZ256rmk, X86::VXORPDZ256rmk, 6187 X86::VPXORQZ256rmk, X86::VPXORDZ256rmk }, 6188 { X86::VXORPSZ256rmkz, X86::VXORPDZ256rmkz, 6189 X86::VPXORQZ256rmkz, X86::VPXORDZ256rmkz }, 6190 { X86::VXORPSZ256rrk, X86::VXORPDZ256rrk, 6191 X86::VPXORQZ256rrk, X86::VPXORDZ256rrk }, 6192 { X86::VXORPSZ256rrkz, X86::VXORPDZ256rrkz, 6193 X86::VPXORQZ256rrkz, X86::VPXORDZ256rrkz }, 6194 { X86::VANDNPSZrmk, X86::VANDNPDZrmk, 6195 X86::VPANDNQZrmk, X86::VPANDNDZrmk }, 6196 { X86::VANDNPSZrmkz, X86::VANDNPDZrmkz, 6197 X86::VPANDNQZrmkz, X86::VPANDNDZrmkz }, 6198 { X86::VANDNPSZrrk, X86::VANDNPDZrrk, 6199 X86::VPANDNQZrrk, X86::VPANDNDZrrk }, 6200 { X86::VANDNPSZrrkz, X86::VANDNPDZrrkz, 6201 X86::VPANDNQZrrkz, X86::VPANDNDZrrkz }, 6202 { X86::VANDPSZrmk, X86::VANDPDZrmk, 6203 X86::VPANDQZrmk, X86::VPANDDZrmk }, 6204 { X86::VANDPSZrmkz, X86::VANDPDZrmkz, 6205 X86::VPANDQZrmkz, X86::VPANDDZrmkz }, 6206 { X86::VANDPSZrrk, X86::VANDPDZrrk, 6207 X86::VPANDQZrrk, X86::VPANDDZrrk }, 6208 { X86::VANDPSZrrkz, X86::VANDPDZrrkz, 6209 X86::VPANDQZrrkz, X86::VPANDDZrrkz }, 6210 { X86::VORPSZrmk, X86::VORPDZrmk, 6211 X86::VPORQZrmk, X86::VPORDZrmk }, 6212 { X86::VORPSZrmkz, X86::VORPDZrmkz, 6213 X86::VPORQZrmkz, X86::VPORDZrmkz }, 6214 { X86::VORPSZrrk, X86::VORPDZrrk, 6215 X86::VPORQZrrk, X86::VPORDZrrk }, 6216 { X86::VORPSZrrkz, X86::VORPDZrrkz, 6217 X86::VPORQZrrkz, X86::VPORDZrrkz }, 6218 { X86::VXORPSZrmk, X86::VXORPDZrmk, 6219 X86::VPXORQZrmk, X86::VPXORDZrmk }, 6220 { X86::VXORPSZrmkz, X86::VXORPDZrmkz, 6221 X86::VPXORQZrmkz, X86::VPXORDZrmkz }, 6222 { X86::VXORPSZrrk, X86::VXORPDZrrk, 6223 X86::VPXORQZrrk, X86::VPXORDZrrk }, 6224 { X86::VXORPSZrrkz, X86::VXORPDZrrkz, 6225 X86::VPXORQZrrkz, X86::VPXORDZrrkz }, 6226 // Broadcast loads can be handled the same as masked operations to avoid 6227 // changing element size. 6228 { X86::VANDNPSZ128rmb, X86::VANDNPDZ128rmb, 6229 X86::VPANDNQZ128rmb, X86::VPANDNDZ128rmb }, 6230 { X86::VANDPSZ128rmb, X86::VANDPDZ128rmb, 6231 X86::VPANDQZ128rmb, X86::VPANDDZ128rmb }, 6232 { X86::VORPSZ128rmb, X86::VORPDZ128rmb, 6233 X86::VPORQZ128rmb, X86::VPORDZ128rmb }, 6234 { X86::VXORPSZ128rmb, X86::VXORPDZ128rmb, 6235 X86::VPXORQZ128rmb, X86::VPXORDZ128rmb }, 6236 { X86::VANDNPSZ256rmb, X86::VANDNPDZ256rmb, 6237 X86::VPANDNQZ256rmb, X86::VPANDNDZ256rmb }, 6238 { X86::VANDPSZ256rmb, X86::VANDPDZ256rmb, 6239 X86::VPANDQZ256rmb, X86::VPANDDZ256rmb }, 6240 { X86::VORPSZ256rmb, X86::VORPDZ256rmb, 6241 X86::VPORQZ256rmb, X86::VPORDZ256rmb }, 6242 { X86::VXORPSZ256rmb, X86::VXORPDZ256rmb, 6243 X86::VPXORQZ256rmb, X86::VPXORDZ256rmb }, 6244 { X86::VANDNPSZrmb, X86::VANDNPDZrmb, 6245 X86::VPANDNQZrmb, X86::VPANDNDZrmb }, 6246 { X86::VANDPSZrmb, X86::VANDPDZrmb, 6247 X86::VPANDQZrmb, X86::VPANDDZrmb }, 6248 { X86::VANDPSZrmb, X86::VANDPDZrmb, 6249 X86::VPANDQZrmb, X86::VPANDDZrmb }, 6250 { X86::VORPSZrmb, X86::VORPDZrmb, 6251 X86::VPORQZrmb, X86::VPORDZrmb }, 6252 { X86::VXORPSZrmb, X86::VXORPDZrmb, 6253 X86::VPXORQZrmb, X86::VPXORDZrmb }, 6254 { X86::VANDNPSZ128rmbk, X86::VANDNPDZ128rmbk, 6255 X86::VPANDNQZ128rmbk, X86::VPANDNDZ128rmbk }, 6256 { X86::VANDPSZ128rmbk, X86::VANDPDZ128rmbk, 6257 X86::VPANDQZ128rmbk, X86::VPANDDZ128rmbk }, 6258 { X86::VORPSZ128rmbk, X86::VORPDZ128rmbk, 6259 X86::VPORQZ128rmbk, X86::VPORDZ128rmbk }, 6260 { X86::VXORPSZ128rmbk, X86::VXORPDZ128rmbk, 6261 X86::VPXORQZ128rmbk, X86::VPXORDZ128rmbk }, 6262 { X86::VANDNPSZ256rmbk, X86::VANDNPDZ256rmbk, 6263 X86::VPANDNQZ256rmbk, X86::VPANDNDZ256rmbk }, 6264 { X86::VANDPSZ256rmbk, X86::VANDPDZ256rmbk, 6265 X86::VPANDQZ256rmbk, X86::VPANDDZ256rmbk }, 6266 { X86::VORPSZ256rmbk, X86::VORPDZ256rmbk, 6267 X86::VPORQZ256rmbk, X86::VPORDZ256rmbk }, 6268 { X86::VXORPSZ256rmbk, X86::VXORPDZ256rmbk, 6269 X86::VPXORQZ256rmbk, X86::VPXORDZ256rmbk }, 6270 { X86::VANDNPSZrmbk, X86::VANDNPDZrmbk, 6271 X86::VPANDNQZrmbk, X86::VPANDNDZrmbk }, 6272 { X86::VANDPSZrmbk, X86::VANDPDZrmbk, 6273 X86::VPANDQZrmbk, X86::VPANDDZrmbk }, 6274 { X86::VANDPSZrmbk, X86::VANDPDZrmbk, 6275 X86::VPANDQZrmbk, X86::VPANDDZrmbk }, 6276 { X86::VORPSZrmbk, X86::VORPDZrmbk, 6277 X86::VPORQZrmbk, X86::VPORDZrmbk }, 6278 { X86::VXORPSZrmbk, X86::VXORPDZrmbk, 6279 X86::VPXORQZrmbk, X86::VPXORDZrmbk }, 6280 { X86::VANDNPSZ128rmbkz,X86::VANDNPDZ128rmbkz, 6281 X86::VPANDNQZ128rmbkz,X86::VPANDNDZ128rmbkz}, 6282 { X86::VANDPSZ128rmbkz, X86::VANDPDZ128rmbkz, 6283 X86::VPANDQZ128rmbkz, X86::VPANDDZ128rmbkz }, 6284 { X86::VORPSZ128rmbkz, X86::VORPDZ128rmbkz, 6285 X86::VPORQZ128rmbkz, X86::VPORDZ128rmbkz }, 6286 { X86::VXORPSZ128rmbkz, X86::VXORPDZ128rmbkz, 6287 X86::VPXORQZ128rmbkz, X86::VPXORDZ128rmbkz }, 6288 { X86::VANDNPSZ256rmbkz,X86::VANDNPDZ256rmbkz, 6289 X86::VPANDNQZ256rmbkz,X86::VPANDNDZ256rmbkz}, 6290 { X86::VANDPSZ256rmbkz, X86::VANDPDZ256rmbkz, 6291 X86::VPANDQZ256rmbkz, X86::VPANDDZ256rmbkz }, 6292 { X86::VORPSZ256rmbkz, X86::VORPDZ256rmbkz, 6293 X86::VPORQZ256rmbkz, X86::VPORDZ256rmbkz }, 6294 { X86::VXORPSZ256rmbkz, X86::VXORPDZ256rmbkz, 6295 X86::VPXORQZ256rmbkz, X86::VPXORDZ256rmbkz }, 6296 { X86::VANDNPSZrmbkz, X86::VANDNPDZrmbkz, 6297 X86::VPANDNQZrmbkz, X86::VPANDNDZrmbkz }, 6298 { X86::VANDPSZrmbkz, X86::VANDPDZrmbkz, 6299 X86::VPANDQZrmbkz, X86::VPANDDZrmbkz }, 6300 { X86::VANDPSZrmbkz, X86::VANDPDZrmbkz, 6301 X86::VPANDQZrmbkz, X86::VPANDDZrmbkz }, 6302 { X86::VORPSZrmbkz, X86::VORPDZrmbkz, 6303 X86::VPORQZrmbkz, X86::VPORDZrmbkz }, 6304 { X86::VXORPSZrmbkz, X86::VXORPDZrmbkz, 6305 X86::VPXORQZrmbkz, X86::VPXORDZrmbkz }, 6306 }; 6307 6308 // NOTE: These should only be used by the custom domain methods. 6309 static const uint16_t ReplaceableCustomInstrs[][3] = { 6310 //PackedSingle PackedDouble PackedInt 6311 { X86::BLENDPSrmi, X86::BLENDPDrmi, X86::PBLENDWrmi }, 6312 { X86::BLENDPSrri, X86::BLENDPDrri, X86::PBLENDWrri }, 6313 { X86::VBLENDPSrmi, X86::VBLENDPDrmi, X86::VPBLENDWrmi }, 6314 { X86::VBLENDPSrri, X86::VBLENDPDrri, X86::VPBLENDWrri }, 6315 { X86::VBLENDPSYrmi, X86::VBLENDPDYrmi, X86::VPBLENDWYrmi }, 6316 { X86::VBLENDPSYrri, X86::VBLENDPDYrri, X86::VPBLENDWYrri }, 6317 }; 6318 static const uint16_t ReplaceableCustomAVX2Instrs[][3] = { 6319 //PackedSingle PackedDouble PackedInt 6320 { X86::VBLENDPSrmi, X86::VBLENDPDrmi, X86::VPBLENDDrmi }, 6321 { X86::VBLENDPSrri, X86::VBLENDPDrri, X86::VPBLENDDrri }, 6322 { X86::VBLENDPSYrmi, X86::VBLENDPDYrmi, X86::VPBLENDDYrmi }, 6323 { X86::VBLENDPSYrri, X86::VBLENDPDYrri, X86::VPBLENDDYrri }, 6324 }; 6325 6326 // Special table for changing EVEX logic instructions to VEX. 6327 // TODO: Should we run EVEX->VEX earlier? 6328 static const uint16_t ReplaceableCustomAVX512LogicInstrs[][4] = { 6329 // Two integer columns for 64-bit and 32-bit elements. 6330 //PackedSingle PackedDouble PackedInt PackedInt 6331 { X86::VANDNPSrm, X86::VANDNPDrm, X86::VPANDNQZ128rm, X86::VPANDNDZ128rm }, 6332 { X86::VANDNPSrr, X86::VANDNPDrr, X86::VPANDNQZ128rr, X86::VPANDNDZ128rr }, 6333 { X86::VANDPSrm, X86::VANDPDrm, X86::VPANDQZ128rm, X86::VPANDDZ128rm }, 6334 { X86::VANDPSrr, X86::VANDPDrr, X86::VPANDQZ128rr, X86::VPANDDZ128rr }, 6335 { X86::VORPSrm, X86::VORPDrm, X86::VPORQZ128rm, X86::VPORDZ128rm }, 6336 { X86::VORPSrr, X86::VORPDrr, X86::VPORQZ128rr, X86::VPORDZ128rr }, 6337 { X86::VXORPSrm, X86::VXORPDrm, X86::VPXORQZ128rm, X86::VPXORDZ128rm }, 6338 { X86::VXORPSrr, X86::VXORPDrr, X86::VPXORQZ128rr, X86::VPXORDZ128rr }, 6339 { X86::VANDNPSYrm, X86::VANDNPDYrm, X86::VPANDNQZ256rm, X86::VPANDNDZ256rm }, 6340 { X86::VANDNPSYrr, X86::VANDNPDYrr, X86::VPANDNQZ256rr, X86::VPANDNDZ256rr }, 6341 { X86::VANDPSYrm, X86::VANDPDYrm, X86::VPANDQZ256rm, X86::VPANDDZ256rm }, 6342 { X86::VANDPSYrr, X86::VANDPDYrr, X86::VPANDQZ256rr, X86::VPANDDZ256rr }, 6343 { X86::VORPSYrm, X86::VORPDYrm, X86::VPORQZ256rm, X86::VPORDZ256rm }, 6344 { X86::VORPSYrr, X86::VORPDYrr, X86::VPORQZ256rr, X86::VPORDZ256rr }, 6345 { X86::VXORPSYrm, X86::VXORPDYrm, X86::VPXORQZ256rm, X86::VPXORDZ256rm }, 6346 { X86::VXORPSYrr, X86::VXORPDYrr, X86::VPXORQZ256rr, X86::VPXORDZ256rr }, 6347 }; 6348 6349 // FIXME: Some shuffle and unpack instructions have equivalents in different 6350 // domains, but they require a bit more work than just switching opcodes. 6351 6352 static const uint16_t *lookup(unsigned opcode, unsigned domain, 6353 ArrayRef<uint16_t[3]> Table) { 6354 for (const uint16_t (&Row)[3] : Table) 6355 if (Row[domain-1] == opcode) 6356 return Row; 6357 return nullptr; 6358 } 6359 6360 static const uint16_t *lookupAVX512(unsigned opcode, unsigned domain, 6361 ArrayRef<uint16_t[4]> Table) { 6362 // If this is the integer domain make sure to check both integer columns. 6363 for (const uint16_t (&Row)[4] : Table) 6364 if (Row[domain-1] == opcode || (domain == 3 && Row[3] == opcode)) 6365 return Row; 6366 return nullptr; 6367 } 6368 6369 // Helper to attempt to widen/narrow blend masks. 6370 static bool AdjustBlendMask(unsigned OldMask, unsigned OldWidth, 6371 unsigned NewWidth, unsigned *pNewMask = nullptr) { 6372 assert(((OldWidth % NewWidth) == 0 || (NewWidth % OldWidth) == 0) && 6373 "Illegal blend mask scale"); 6374 unsigned NewMask = 0; 6375 6376 if ((OldWidth % NewWidth) == 0) { 6377 unsigned Scale = OldWidth / NewWidth; 6378 unsigned SubMask = (1u << Scale) - 1; 6379 for (unsigned i = 0; i != NewWidth; ++i) { 6380 unsigned Sub = (OldMask >> (i * Scale)) & SubMask; 6381 if (Sub == SubMask) 6382 NewMask |= (1u << i); 6383 else if (Sub != 0x0) 6384 return false; 6385 } 6386 } else { 6387 unsigned Scale = NewWidth / OldWidth; 6388 unsigned SubMask = (1u << Scale) - 1; 6389 for (unsigned i = 0; i != OldWidth; ++i) { 6390 if (OldMask & (1 << i)) { 6391 NewMask |= (SubMask << (i * Scale)); 6392 } 6393 } 6394 } 6395 6396 if (pNewMask) 6397 *pNewMask = NewMask; 6398 return true; 6399 } 6400 6401 uint16_t X86InstrInfo::getExecutionDomainCustom(const MachineInstr &MI) const { 6402 unsigned Opcode = MI.getOpcode(); 6403 unsigned NumOperands = MI.getDesc().getNumOperands(); 6404 6405 auto GetBlendDomains = [&](unsigned ImmWidth, bool Is256) { 6406 uint16_t validDomains = 0; 6407 if (MI.getOperand(NumOperands - 1).isImm()) { 6408 unsigned Imm = MI.getOperand(NumOperands - 1).getImm(); 6409 if (AdjustBlendMask(Imm, ImmWidth, Is256 ? 8 : 4)) 6410 validDomains |= 0x2; // PackedSingle 6411 if (AdjustBlendMask(Imm, ImmWidth, Is256 ? 4 : 2)) 6412 validDomains |= 0x4; // PackedDouble 6413 if (!Is256 || Subtarget.hasAVX2()) 6414 validDomains |= 0x8; // PackedInt 6415 } 6416 return validDomains; 6417 }; 6418 6419 switch (Opcode) { 6420 case X86::BLENDPDrmi: 6421 case X86::BLENDPDrri: 6422 case X86::VBLENDPDrmi: 6423 case X86::VBLENDPDrri: 6424 return GetBlendDomains(2, false); 6425 case X86::VBLENDPDYrmi: 6426 case X86::VBLENDPDYrri: 6427 return GetBlendDomains(4, true); 6428 case X86::BLENDPSrmi: 6429 case X86::BLENDPSrri: 6430 case X86::VBLENDPSrmi: 6431 case X86::VBLENDPSrri: 6432 case X86::VPBLENDDrmi: 6433 case X86::VPBLENDDrri: 6434 return GetBlendDomains(4, false); 6435 case X86::VBLENDPSYrmi: 6436 case X86::VBLENDPSYrri: 6437 case X86::VPBLENDDYrmi: 6438 case X86::VPBLENDDYrri: 6439 return GetBlendDomains(8, true); 6440 case X86::PBLENDWrmi: 6441 case X86::PBLENDWrri: 6442 case X86::VPBLENDWrmi: 6443 case X86::VPBLENDWrri: 6444 // Treat VPBLENDWY as a 128-bit vector as it repeats the lo/hi masks. 6445 case X86::VPBLENDWYrmi: 6446 case X86::VPBLENDWYrri: 6447 return GetBlendDomains(8, false); 6448 case X86::VPANDDZ128rr: case X86::VPANDDZ128rm: 6449 case X86::VPANDDZ256rr: case X86::VPANDDZ256rm: 6450 case X86::VPANDQZ128rr: case X86::VPANDQZ128rm: 6451 case X86::VPANDQZ256rr: case X86::VPANDQZ256rm: 6452 case X86::VPANDNDZ128rr: case X86::VPANDNDZ128rm: 6453 case X86::VPANDNDZ256rr: case X86::VPANDNDZ256rm: 6454 case X86::VPANDNQZ128rr: case X86::VPANDNQZ128rm: 6455 case X86::VPANDNQZ256rr: case X86::VPANDNQZ256rm: 6456 case X86::VPORDZ128rr: case X86::VPORDZ128rm: 6457 case X86::VPORDZ256rr: case X86::VPORDZ256rm: 6458 case X86::VPORQZ128rr: case X86::VPORQZ128rm: 6459 case X86::VPORQZ256rr: case X86::VPORQZ256rm: 6460 case X86::VPXORDZ128rr: case X86::VPXORDZ128rm: 6461 case X86::VPXORDZ256rr: case X86::VPXORDZ256rm: 6462 case X86::VPXORQZ128rr: case X86::VPXORQZ128rm: 6463 case X86::VPXORQZ256rr: case X86::VPXORQZ256rm: 6464 // If we don't have DQI see if we can still switch from an EVEX integer 6465 // instruction to a VEX floating point instruction. 6466 if (Subtarget.hasDQI()) 6467 return 0; 6468 6469 if (RI.getEncodingValue(MI.getOperand(0).getReg()) >= 16) 6470 return 0; 6471 if (RI.getEncodingValue(MI.getOperand(1).getReg()) >= 16) 6472 return 0; 6473 // Register forms will have 3 operands. Memory form will have more. 6474 if (NumOperands == 3 && 6475 RI.getEncodingValue(MI.getOperand(2).getReg()) >= 16) 6476 return 0; 6477 6478 // All domains are valid. 6479 return 0xe; 6480 } 6481 return 0; 6482 } 6483 6484 bool X86InstrInfo::setExecutionDomainCustom(MachineInstr &MI, 6485 unsigned Domain) const { 6486 assert(Domain > 0 && Domain < 4 && "Invalid execution domain"); 6487 uint16_t dom = (MI.getDesc().TSFlags >> X86II::SSEDomainShift) & 3; 6488 assert(dom && "Not an SSE instruction"); 6489 6490 unsigned Opcode = MI.getOpcode(); 6491 unsigned NumOperands = MI.getDesc().getNumOperands(); 6492 6493 auto SetBlendDomain = [&](unsigned ImmWidth, bool Is256) { 6494 if (MI.getOperand(NumOperands - 1).isImm()) { 6495 unsigned Imm = MI.getOperand(NumOperands - 1).getImm() & 255; 6496 Imm = (ImmWidth == 16 ? ((Imm << 8) | Imm) : Imm); 6497 unsigned NewImm = Imm; 6498 6499 const uint16_t *table = lookup(Opcode, dom, ReplaceableCustomInstrs); 6500 if (!table) 6501 table = lookup(Opcode, dom, ReplaceableCustomAVX2Instrs); 6502 6503 if (Domain == 1) { // PackedSingle 6504 AdjustBlendMask(Imm, ImmWidth, Is256 ? 8 : 4, &NewImm); 6505 } else if (Domain == 2) { // PackedDouble 6506 AdjustBlendMask(Imm, ImmWidth, Is256 ? 4 : 2, &NewImm); 6507 } else if (Domain == 3) { // PackedInt 6508 if (Subtarget.hasAVX2()) { 6509 // If we are already VPBLENDW use that, else use VPBLENDD. 6510 if ((ImmWidth / (Is256 ? 2 : 1)) != 8) { 6511 table = lookup(Opcode, dom, ReplaceableCustomAVX2Instrs); 6512 AdjustBlendMask(Imm, ImmWidth, Is256 ? 8 : 4, &NewImm); 6513 } 6514 } else { 6515 assert(!Is256 && "128-bit vector expected"); 6516 AdjustBlendMask(Imm, ImmWidth, 8, &NewImm); 6517 } 6518 } 6519 6520 assert(table && table[Domain - 1] && "Unknown domain op"); 6521 MI.setDesc(get(table[Domain - 1])); 6522 MI.getOperand(NumOperands - 1).setImm(NewImm & 255); 6523 } 6524 return true; 6525 }; 6526 6527 switch (Opcode) { 6528 case X86::BLENDPDrmi: 6529 case X86::BLENDPDrri: 6530 case X86::VBLENDPDrmi: 6531 case X86::VBLENDPDrri: 6532 return SetBlendDomain(2, false); 6533 case X86::VBLENDPDYrmi: 6534 case X86::VBLENDPDYrri: 6535 return SetBlendDomain(4, true); 6536 case X86::BLENDPSrmi: 6537 case X86::BLENDPSrri: 6538 case X86::VBLENDPSrmi: 6539 case X86::VBLENDPSrri: 6540 case X86::VPBLENDDrmi: 6541 case X86::VPBLENDDrri: 6542 return SetBlendDomain(4, false); 6543 case X86::VBLENDPSYrmi: 6544 case X86::VBLENDPSYrri: 6545 case X86::VPBLENDDYrmi: 6546 case X86::VPBLENDDYrri: 6547 return SetBlendDomain(8, true); 6548 case X86::PBLENDWrmi: 6549 case X86::PBLENDWrri: 6550 case X86::VPBLENDWrmi: 6551 case X86::VPBLENDWrri: 6552 return SetBlendDomain(8, false); 6553 case X86::VPBLENDWYrmi: 6554 case X86::VPBLENDWYrri: 6555 return SetBlendDomain(16, true); 6556 case X86::VPANDDZ128rr: case X86::VPANDDZ128rm: 6557 case X86::VPANDDZ256rr: case X86::VPANDDZ256rm: 6558 case X86::VPANDQZ128rr: case X86::VPANDQZ128rm: 6559 case X86::VPANDQZ256rr: case X86::VPANDQZ256rm: 6560 case X86::VPANDNDZ128rr: case X86::VPANDNDZ128rm: 6561 case X86::VPANDNDZ256rr: case X86::VPANDNDZ256rm: 6562 case X86::VPANDNQZ128rr: case X86::VPANDNQZ128rm: 6563 case X86::VPANDNQZ256rr: case X86::VPANDNQZ256rm: 6564 case X86::VPORDZ128rr: case X86::VPORDZ128rm: 6565 case X86::VPORDZ256rr: case X86::VPORDZ256rm: 6566 case X86::VPORQZ128rr: case X86::VPORQZ128rm: 6567 case X86::VPORQZ256rr: case X86::VPORQZ256rm: 6568 case X86::VPXORDZ128rr: case X86::VPXORDZ128rm: 6569 case X86::VPXORDZ256rr: case X86::VPXORDZ256rm: 6570 case X86::VPXORQZ128rr: case X86::VPXORQZ128rm: 6571 case X86::VPXORQZ256rr: case X86::VPXORQZ256rm: { 6572 // Without DQI, convert EVEX instructions to VEX instructions. 6573 if (Subtarget.hasDQI()) 6574 return false; 6575 6576 const uint16_t *table = lookupAVX512(MI.getOpcode(), dom, 6577 ReplaceableCustomAVX512LogicInstrs); 6578 assert(table && "Instruction not found in table?"); 6579 // Don't change integer Q instructions to D instructions and 6580 // use D intructions if we started with a PS instruction. 6581 if (Domain == 3 && (dom == 1 || table[3] == MI.getOpcode())) 6582 Domain = 4; 6583 MI.setDesc(get(table[Domain - 1])); 6584 return true; 6585 } 6586 } 6587 return false; 6588 } 6589 6590 std::pair<uint16_t, uint16_t> 6591 X86InstrInfo::getExecutionDomain(const MachineInstr &MI) const { 6592 uint16_t domain = (MI.getDesc().TSFlags >> X86II::SSEDomainShift) & 3; 6593 unsigned opcode = MI.getOpcode(); 6594 uint16_t validDomains = 0; 6595 if (domain) { 6596 // Attempt to match for custom instructions. 6597 validDomains = getExecutionDomainCustom(MI); 6598 if (validDomains) 6599 return std::make_pair(domain, validDomains); 6600 6601 if (lookup(opcode, domain, ReplaceableInstrs)) { 6602 validDomains = 0xe; 6603 } else if (lookup(opcode, domain, ReplaceableInstrsAVX2)) { 6604 validDomains = Subtarget.hasAVX2() ? 0xe : 0x6; 6605 } else if (lookup(opcode, domain, ReplaceableInstrsAVX2InsertExtract)) { 6606 // Insert/extract instructions should only effect domain if AVX2 6607 // is enabled. 6608 if (!Subtarget.hasAVX2()) 6609 return std::make_pair(0, 0); 6610 validDomains = 0xe; 6611 } else if (lookupAVX512(opcode, domain, ReplaceableInstrsAVX512)) { 6612 validDomains = 0xe; 6613 } else if (Subtarget.hasDQI() && lookupAVX512(opcode, domain, 6614 ReplaceableInstrsAVX512DQ)) { 6615 validDomains = 0xe; 6616 } else if (Subtarget.hasDQI()) { 6617 if (const uint16_t *table = lookupAVX512(opcode, domain, 6618 ReplaceableInstrsAVX512DQMasked)) { 6619 if (domain == 1 || (domain == 3 && table[3] == opcode)) 6620 validDomains = 0xa; 6621 else 6622 validDomains = 0xc; 6623 } 6624 } 6625 } 6626 return std::make_pair(domain, validDomains); 6627 } 6628 6629 void X86InstrInfo::setExecutionDomain(MachineInstr &MI, unsigned Domain) const { 6630 assert(Domain>0 && Domain<4 && "Invalid execution domain"); 6631 uint16_t dom = (MI.getDesc().TSFlags >> X86II::SSEDomainShift) & 3; 6632 assert(dom && "Not an SSE instruction"); 6633 6634 // Attempt to match for custom instructions. 6635 if (setExecutionDomainCustom(MI, Domain)) 6636 return; 6637 6638 const uint16_t *table = lookup(MI.getOpcode(), dom, ReplaceableInstrs); 6639 if (!table) { // try the other table 6640 assert((Subtarget.hasAVX2() || Domain < 3) && 6641 "256-bit vector operations only available in AVX2"); 6642 table = lookup(MI.getOpcode(), dom, ReplaceableInstrsAVX2); 6643 } 6644 if (!table) { // try the other table 6645 assert(Subtarget.hasAVX2() && 6646 "256-bit insert/extract only available in AVX2"); 6647 table = lookup(MI.getOpcode(), dom, ReplaceableInstrsAVX2InsertExtract); 6648 } 6649 if (!table) { // try the AVX512 table 6650 assert(Subtarget.hasAVX512() && "Requires AVX-512"); 6651 table = lookupAVX512(MI.getOpcode(), dom, ReplaceableInstrsAVX512); 6652 // Don't change integer Q instructions to D instructions. 6653 if (table && Domain == 3 && table[3] == MI.getOpcode()) 6654 Domain = 4; 6655 } 6656 if (!table) { // try the AVX512DQ table 6657 assert((Subtarget.hasDQI() || Domain >= 3) && "Requires AVX-512DQ"); 6658 table = lookupAVX512(MI.getOpcode(), dom, ReplaceableInstrsAVX512DQ); 6659 // Don't change integer Q instructions to D instructions and 6660 // use D intructions if we started with a PS instruction. 6661 if (table && Domain == 3 && (dom == 1 || table[3] == MI.getOpcode())) 6662 Domain = 4; 6663 } 6664 if (!table) { // try the AVX512DQMasked table 6665 assert((Subtarget.hasDQI() || Domain >= 3) && "Requires AVX-512DQ"); 6666 table = lookupAVX512(MI.getOpcode(), dom, ReplaceableInstrsAVX512DQMasked); 6667 if (table && Domain == 3 && (dom == 1 || table[3] == MI.getOpcode())) 6668 Domain = 4; 6669 } 6670 assert(table && "Cannot change domain"); 6671 MI.setDesc(get(table[Domain - 1])); 6672 } 6673 6674 /// Return the noop instruction to use for a noop. 6675 void X86InstrInfo::getNoop(MCInst &NopInst) const { 6676 NopInst.setOpcode(X86::NOOP); 6677 } 6678 6679 bool X86InstrInfo::isHighLatencyDef(int opc) const { 6680 switch (opc) { 6681 default: return false; 6682 case X86::DIVPDrm: 6683 case X86::DIVPDrr: 6684 case X86::DIVPSrm: 6685 case X86::DIVPSrr: 6686 case X86::DIVSDrm: 6687 case X86::DIVSDrm_Int: 6688 case X86::DIVSDrr: 6689 case X86::DIVSDrr_Int: 6690 case X86::DIVSSrm: 6691 case X86::DIVSSrm_Int: 6692 case X86::DIVSSrr: 6693 case X86::DIVSSrr_Int: 6694 case X86::SQRTPDm: 6695 case X86::SQRTPDr: 6696 case X86::SQRTPSm: 6697 case X86::SQRTPSr: 6698 case X86::SQRTSDm: 6699 case X86::SQRTSDm_Int: 6700 case X86::SQRTSDr: 6701 case X86::SQRTSDr_Int: 6702 case X86::SQRTSSm: 6703 case X86::SQRTSSm_Int: 6704 case X86::SQRTSSr: 6705 case X86::SQRTSSr_Int: 6706 // AVX instructions with high latency 6707 case X86::VDIVPDrm: 6708 case X86::VDIVPDrr: 6709 case X86::VDIVPDYrm: 6710 case X86::VDIVPDYrr: 6711 case X86::VDIVPSrm: 6712 case X86::VDIVPSrr: 6713 case X86::VDIVPSYrm: 6714 case X86::VDIVPSYrr: 6715 case X86::VDIVSDrm: 6716 case X86::VDIVSDrm_Int: 6717 case X86::VDIVSDrr: 6718 case X86::VDIVSDrr_Int: 6719 case X86::VDIVSSrm: 6720 case X86::VDIVSSrm_Int: 6721 case X86::VDIVSSrr: 6722 case X86::VDIVSSrr_Int: 6723 case X86::VSQRTPDm: 6724 case X86::VSQRTPDr: 6725 case X86::VSQRTPDYm: 6726 case X86::VSQRTPDYr: 6727 case X86::VSQRTPSm: 6728 case X86::VSQRTPSr: 6729 case X86::VSQRTPSYm: 6730 case X86::VSQRTPSYr: 6731 case X86::VSQRTSDm: 6732 case X86::VSQRTSDm_Int: 6733 case X86::VSQRTSDr: 6734 case X86::VSQRTSDr_Int: 6735 case X86::VSQRTSSm: 6736 case X86::VSQRTSSm_Int: 6737 case X86::VSQRTSSr: 6738 case X86::VSQRTSSr_Int: 6739 // AVX512 instructions with high latency 6740 case X86::VDIVPDZ128rm: 6741 case X86::VDIVPDZ128rmb: 6742 case X86::VDIVPDZ128rmbk: 6743 case X86::VDIVPDZ128rmbkz: 6744 case X86::VDIVPDZ128rmk: 6745 case X86::VDIVPDZ128rmkz: 6746 case X86::VDIVPDZ128rr: 6747 case X86::VDIVPDZ128rrk: 6748 case X86::VDIVPDZ128rrkz: 6749 case X86::VDIVPDZ256rm: 6750 case X86::VDIVPDZ256rmb: 6751 case X86::VDIVPDZ256rmbk: 6752 case X86::VDIVPDZ256rmbkz: 6753 case X86::VDIVPDZ256rmk: 6754 case X86::VDIVPDZ256rmkz: 6755 case X86::VDIVPDZ256rr: 6756 case X86::VDIVPDZ256rrk: 6757 case X86::VDIVPDZ256rrkz: 6758 case X86::VDIVPDZrrb: 6759 case X86::VDIVPDZrrbk: 6760 case X86::VDIVPDZrrbkz: 6761 case X86::VDIVPDZrm: 6762 case X86::VDIVPDZrmb: 6763 case X86::VDIVPDZrmbk: 6764 case X86::VDIVPDZrmbkz: 6765 case X86::VDIVPDZrmk: 6766 case X86::VDIVPDZrmkz: 6767 case X86::VDIVPDZrr: 6768 case X86::VDIVPDZrrk: 6769 case X86::VDIVPDZrrkz: 6770 case X86::VDIVPSZ128rm: 6771 case X86::VDIVPSZ128rmb: 6772 case X86::VDIVPSZ128rmbk: 6773 case X86::VDIVPSZ128rmbkz: 6774 case X86::VDIVPSZ128rmk: 6775 case X86::VDIVPSZ128rmkz: 6776 case X86::VDIVPSZ128rr: 6777 case X86::VDIVPSZ128rrk: 6778 case X86::VDIVPSZ128rrkz: 6779 case X86::VDIVPSZ256rm: 6780 case X86::VDIVPSZ256rmb: 6781 case X86::VDIVPSZ256rmbk: 6782 case X86::VDIVPSZ256rmbkz: 6783 case X86::VDIVPSZ256rmk: 6784 case X86::VDIVPSZ256rmkz: 6785 case X86::VDIVPSZ256rr: 6786 case X86::VDIVPSZ256rrk: 6787 case X86::VDIVPSZ256rrkz: 6788 case X86::VDIVPSZrrb: 6789 case X86::VDIVPSZrrbk: 6790 case X86::VDIVPSZrrbkz: 6791 case X86::VDIVPSZrm: 6792 case X86::VDIVPSZrmb: 6793 case X86::VDIVPSZrmbk: 6794 case X86::VDIVPSZrmbkz: 6795 case X86::VDIVPSZrmk: 6796 case X86::VDIVPSZrmkz: 6797 case X86::VDIVPSZrr: 6798 case X86::VDIVPSZrrk: 6799 case X86::VDIVPSZrrkz: 6800 case X86::VDIVSDZrm: 6801 case X86::VDIVSDZrr: 6802 case X86::VDIVSDZrm_Int: 6803 case X86::VDIVSDZrm_Intk: 6804 case X86::VDIVSDZrm_Intkz: 6805 case X86::VDIVSDZrr_Int: 6806 case X86::VDIVSDZrr_Intk: 6807 case X86::VDIVSDZrr_Intkz: 6808 case X86::VDIVSDZrrb_Int: 6809 case X86::VDIVSDZrrb_Intk: 6810 case X86::VDIVSDZrrb_Intkz: 6811 case X86::VDIVSSZrm: 6812 case X86::VDIVSSZrr: 6813 case X86::VDIVSSZrm_Int: 6814 case X86::VDIVSSZrm_Intk: 6815 case X86::VDIVSSZrm_Intkz: 6816 case X86::VDIVSSZrr_Int: 6817 case X86::VDIVSSZrr_Intk: 6818 case X86::VDIVSSZrr_Intkz: 6819 case X86::VDIVSSZrrb_Int: 6820 case X86::VDIVSSZrrb_Intk: 6821 case X86::VDIVSSZrrb_Intkz: 6822 case X86::VSQRTPDZ128m: 6823 case X86::VSQRTPDZ128mb: 6824 case X86::VSQRTPDZ128mbk: 6825 case X86::VSQRTPDZ128mbkz: 6826 case X86::VSQRTPDZ128mk: 6827 case X86::VSQRTPDZ128mkz: 6828 case X86::VSQRTPDZ128r: 6829 case X86::VSQRTPDZ128rk: 6830 case X86::VSQRTPDZ128rkz: 6831 case X86::VSQRTPDZ256m: 6832 case X86::VSQRTPDZ256mb: 6833 case X86::VSQRTPDZ256mbk: 6834 case X86::VSQRTPDZ256mbkz: 6835 case X86::VSQRTPDZ256mk: 6836 case X86::VSQRTPDZ256mkz: 6837 case X86::VSQRTPDZ256r: 6838 case X86::VSQRTPDZ256rk: 6839 case X86::VSQRTPDZ256rkz: 6840 case X86::VSQRTPDZm: 6841 case X86::VSQRTPDZmb: 6842 case X86::VSQRTPDZmbk: 6843 case X86::VSQRTPDZmbkz: 6844 case X86::VSQRTPDZmk: 6845 case X86::VSQRTPDZmkz: 6846 case X86::VSQRTPDZr: 6847 case X86::VSQRTPDZrb: 6848 case X86::VSQRTPDZrbk: 6849 case X86::VSQRTPDZrbkz: 6850 case X86::VSQRTPDZrk: 6851 case X86::VSQRTPDZrkz: 6852 case X86::VSQRTPSZ128m: 6853 case X86::VSQRTPSZ128mb: 6854 case X86::VSQRTPSZ128mbk: 6855 case X86::VSQRTPSZ128mbkz: 6856 case X86::VSQRTPSZ128mk: 6857 case X86::VSQRTPSZ128mkz: 6858 case X86::VSQRTPSZ128r: 6859 case X86::VSQRTPSZ128rk: 6860 case X86::VSQRTPSZ128rkz: 6861 case X86::VSQRTPSZ256m: 6862 case X86::VSQRTPSZ256mb: 6863 case X86::VSQRTPSZ256mbk: 6864 case X86::VSQRTPSZ256mbkz: 6865 case X86::VSQRTPSZ256mk: 6866 case X86::VSQRTPSZ256mkz: 6867 case X86::VSQRTPSZ256r: 6868 case X86::VSQRTPSZ256rk: 6869 case X86::VSQRTPSZ256rkz: 6870 case X86::VSQRTPSZm: 6871 case X86::VSQRTPSZmb: 6872 case X86::VSQRTPSZmbk: 6873 case X86::VSQRTPSZmbkz: 6874 case X86::VSQRTPSZmk: 6875 case X86::VSQRTPSZmkz: 6876 case X86::VSQRTPSZr: 6877 case X86::VSQRTPSZrb: 6878 case X86::VSQRTPSZrbk: 6879 case X86::VSQRTPSZrbkz: 6880 case X86::VSQRTPSZrk: 6881 case X86::VSQRTPSZrkz: 6882 case X86::VSQRTSDZm: 6883 case X86::VSQRTSDZm_Int: 6884 case X86::VSQRTSDZm_Intk: 6885 case X86::VSQRTSDZm_Intkz: 6886 case X86::VSQRTSDZr: 6887 case X86::VSQRTSDZr_Int: 6888 case X86::VSQRTSDZr_Intk: 6889 case X86::VSQRTSDZr_Intkz: 6890 case X86::VSQRTSDZrb_Int: 6891 case X86::VSQRTSDZrb_Intk: 6892 case X86::VSQRTSDZrb_Intkz: 6893 case X86::VSQRTSSZm: 6894 case X86::VSQRTSSZm_Int: 6895 case X86::VSQRTSSZm_Intk: 6896 case X86::VSQRTSSZm_Intkz: 6897 case X86::VSQRTSSZr: 6898 case X86::VSQRTSSZr_Int: 6899 case X86::VSQRTSSZr_Intk: 6900 case X86::VSQRTSSZr_Intkz: 6901 case X86::VSQRTSSZrb_Int: 6902 case X86::VSQRTSSZrb_Intk: 6903 case X86::VSQRTSSZrb_Intkz: 6904 6905 case X86::VGATHERDPDYrm: 6906 case X86::VGATHERDPDZ128rm: 6907 case X86::VGATHERDPDZ256rm: 6908 case X86::VGATHERDPDZrm: 6909 case X86::VGATHERDPDrm: 6910 case X86::VGATHERDPSYrm: 6911 case X86::VGATHERDPSZ128rm: 6912 case X86::VGATHERDPSZ256rm: 6913 case X86::VGATHERDPSZrm: 6914 case X86::VGATHERDPSrm: 6915 case X86::VGATHERPF0DPDm: 6916 case X86::VGATHERPF0DPSm: 6917 case X86::VGATHERPF0QPDm: 6918 case X86::VGATHERPF0QPSm: 6919 case X86::VGATHERPF1DPDm: 6920 case X86::VGATHERPF1DPSm: 6921 case X86::VGATHERPF1QPDm: 6922 case X86::VGATHERPF1QPSm: 6923 case X86::VGATHERQPDYrm: 6924 case X86::VGATHERQPDZ128rm: 6925 case X86::VGATHERQPDZ256rm: 6926 case X86::VGATHERQPDZrm: 6927 case X86::VGATHERQPDrm: 6928 case X86::VGATHERQPSYrm: 6929 case X86::VGATHERQPSZ128rm: 6930 case X86::VGATHERQPSZ256rm: 6931 case X86::VGATHERQPSZrm: 6932 case X86::VGATHERQPSrm: 6933 case X86::VPGATHERDDYrm: 6934 case X86::VPGATHERDDZ128rm: 6935 case X86::VPGATHERDDZ256rm: 6936 case X86::VPGATHERDDZrm: 6937 case X86::VPGATHERDDrm: 6938 case X86::VPGATHERDQYrm: 6939 case X86::VPGATHERDQZ128rm: 6940 case X86::VPGATHERDQZ256rm: 6941 case X86::VPGATHERDQZrm: 6942 case X86::VPGATHERDQrm: 6943 case X86::VPGATHERQDYrm: 6944 case X86::VPGATHERQDZ128rm: 6945 case X86::VPGATHERQDZ256rm: 6946 case X86::VPGATHERQDZrm: 6947 case X86::VPGATHERQDrm: 6948 case X86::VPGATHERQQYrm: 6949 case X86::VPGATHERQQZ128rm: 6950 case X86::VPGATHERQQZ256rm: 6951 case X86::VPGATHERQQZrm: 6952 case X86::VPGATHERQQrm: 6953 case X86::VSCATTERDPDZ128mr: 6954 case X86::VSCATTERDPDZ256mr: 6955 case X86::VSCATTERDPDZmr: 6956 case X86::VSCATTERDPSZ128mr: 6957 case X86::VSCATTERDPSZ256mr: 6958 case X86::VSCATTERDPSZmr: 6959 case X86::VSCATTERPF0DPDm: 6960 case X86::VSCATTERPF0DPSm: 6961 case X86::VSCATTERPF0QPDm: 6962 case X86::VSCATTERPF0QPSm: 6963 case X86::VSCATTERPF1DPDm: 6964 case X86::VSCATTERPF1DPSm: 6965 case X86::VSCATTERPF1QPDm: 6966 case X86::VSCATTERPF1QPSm: 6967 case X86::VSCATTERQPDZ128mr: 6968 case X86::VSCATTERQPDZ256mr: 6969 case X86::VSCATTERQPDZmr: 6970 case X86::VSCATTERQPSZ128mr: 6971 case X86::VSCATTERQPSZ256mr: 6972 case X86::VSCATTERQPSZmr: 6973 case X86::VPSCATTERDDZ128mr: 6974 case X86::VPSCATTERDDZ256mr: 6975 case X86::VPSCATTERDDZmr: 6976 case X86::VPSCATTERDQZ128mr: 6977 case X86::VPSCATTERDQZ256mr: 6978 case X86::VPSCATTERDQZmr: 6979 case X86::VPSCATTERQDZ128mr: 6980 case X86::VPSCATTERQDZ256mr: 6981 case X86::VPSCATTERQDZmr: 6982 case X86::VPSCATTERQQZ128mr: 6983 case X86::VPSCATTERQQZ256mr: 6984 case X86::VPSCATTERQQZmr: 6985 return true; 6986 } 6987 } 6988 6989 bool X86InstrInfo::hasHighOperandLatency(const TargetSchedModel &SchedModel, 6990 const MachineRegisterInfo *MRI, 6991 const MachineInstr &DefMI, 6992 unsigned DefIdx, 6993 const MachineInstr &UseMI, 6994 unsigned UseIdx) const { 6995 return isHighLatencyDef(DefMI.getOpcode()); 6996 } 6997 6998 bool X86InstrInfo::hasReassociableOperands(const MachineInstr &Inst, 6999 const MachineBasicBlock *MBB) const { 7000 assert((Inst.getNumOperands() == 3 || Inst.getNumOperands() == 4) && 7001 "Reassociation needs binary operators"); 7002 7003 // Integer binary math/logic instructions have a third source operand: 7004 // the EFLAGS register. That operand must be both defined here and never 7005 // used; ie, it must be dead. If the EFLAGS operand is live, then we can 7006 // not change anything because rearranging the operands could affect other 7007 // instructions that depend on the exact status flags (zero, sign, etc.) 7008 // that are set by using these particular operands with this operation. 7009 if (Inst.getNumOperands() == 4) { 7010 assert(Inst.getOperand(3).isReg() && 7011 Inst.getOperand(3).getReg() == X86::EFLAGS && 7012 "Unexpected operand in reassociable instruction"); 7013 if (!Inst.getOperand(3).isDead()) 7014 return false; 7015 } 7016 7017 return TargetInstrInfo::hasReassociableOperands(Inst, MBB); 7018 } 7019 7020 // TODO: There are many more machine instruction opcodes to match: 7021 // 1. Other data types (integer, vectors) 7022 // 2. Other math / logic operations (xor, or) 7023 // 3. Other forms of the same operation (intrinsics and other variants) 7024 bool X86InstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst) const { 7025 switch (Inst.getOpcode()) { 7026 case X86::AND8rr: 7027 case X86::AND16rr: 7028 case X86::AND32rr: 7029 case X86::AND64rr: 7030 case X86::OR8rr: 7031 case X86::OR16rr: 7032 case X86::OR32rr: 7033 case X86::OR64rr: 7034 case X86::XOR8rr: 7035 case X86::XOR16rr: 7036 case X86::XOR32rr: 7037 case X86::XOR64rr: 7038 case X86::IMUL16rr: 7039 case X86::IMUL32rr: 7040 case X86::IMUL64rr: 7041 case X86::PANDrr: 7042 case X86::PORrr: 7043 case X86::PXORrr: 7044 case X86::ANDPDrr: 7045 case X86::ANDPSrr: 7046 case X86::ORPDrr: 7047 case X86::ORPSrr: 7048 case X86::XORPDrr: 7049 case X86::XORPSrr: 7050 case X86::PADDBrr: 7051 case X86::PADDWrr: 7052 case X86::PADDDrr: 7053 case X86::PADDQrr: 7054 case X86::VPANDrr: 7055 case X86::VPANDYrr: 7056 case X86::VPANDDZ128rr: 7057 case X86::VPANDDZ256rr: 7058 case X86::VPANDDZrr: 7059 case X86::VPANDQZ128rr: 7060 case X86::VPANDQZ256rr: 7061 case X86::VPANDQZrr: 7062 case X86::VPORrr: 7063 case X86::VPORYrr: 7064 case X86::VPORDZ128rr: 7065 case X86::VPORDZ256rr: 7066 case X86::VPORDZrr: 7067 case X86::VPORQZ128rr: 7068 case X86::VPORQZ256rr: 7069 case X86::VPORQZrr: 7070 case X86::VPXORrr: 7071 case X86::VPXORYrr: 7072 case X86::VPXORDZ128rr: 7073 case X86::VPXORDZ256rr: 7074 case X86::VPXORDZrr: 7075 case X86::VPXORQZ128rr: 7076 case X86::VPXORQZ256rr: 7077 case X86::VPXORQZrr: 7078 case X86::VANDPDrr: 7079 case X86::VANDPSrr: 7080 case X86::VANDPDYrr: 7081 case X86::VANDPSYrr: 7082 case X86::VANDPDZ128rr: 7083 case X86::VANDPSZ128rr: 7084 case X86::VANDPDZ256rr: 7085 case X86::VANDPSZ256rr: 7086 case X86::VANDPDZrr: 7087 case X86::VANDPSZrr: 7088 case X86::VORPDrr: 7089 case X86::VORPSrr: 7090 case X86::VORPDYrr: 7091 case X86::VORPSYrr: 7092 case X86::VORPDZ128rr: 7093 case X86::VORPSZ128rr: 7094 case X86::VORPDZ256rr: 7095 case X86::VORPSZ256rr: 7096 case X86::VORPDZrr: 7097 case X86::VORPSZrr: 7098 case X86::VXORPDrr: 7099 case X86::VXORPSrr: 7100 case X86::VXORPDYrr: 7101 case X86::VXORPSYrr: 7102 case X86::VXORPDZ128rr: 7103 case X86::VXORPSZ128rr: 7104 case X86::VXORPDZ256rr: 7105 case X86::VXORPSZ256rr: 7106 case X86::VXORPDZrr: 7107 case X86::VXORPSZrr: 7108 case X86::KADDBrr: 7109 case X86::KADDWrr: 7110 case X86::KADDDrr: 7111 case X86::KADDQrr: 7112 case X86::KANDBrr: 7113 case X86::KANDWrr: 7114 case X86::KANDDrr: 7115 case X86::KANDQrr: 7116 case X86::KORBrr: 7117 case X86::KORWrr: 7118 case X86::KORDrr: 7119 case X86::KORQrr: 7120 case X86::KXORBrr: 7121 case X86::KXORWrr: 7122 case X86::KXORDrr: 7123 case X86::KXORQrr: 7124 case X86::VPADDBrr: 7125 case X86::VPADDWrr: 7126 case X86::VPADDDrr: 7127 case X86::VPADDQrr: 7128 case X86::VPADDBYrr: 7129 case X86::VPADDWYrr: 7130 case X86::VPADDDYrr: 7131 case X86::VPADDQYrr: 7132 case X86::VPADDBZ128rr: 7133 case X86::VPADDWZ128rr: 7134 case X86::VPADDDZ128rr: 7135 case X86::VPADDQZ128rr: 7136 case X86::VPADDBZ256rr: 7137 case X86::VPADDWZ256rr: 7138 case X86::VPADDDZ256rr: 7139 case X86::VPADDQZ256rr: 7140 case X86::VPADDBZrr: 7141 case X86::VPADDWZrr: 7142 case X86::VPADDDZrr: 7143 case X86::VPADDQZrr: 7144 case X86::VPMULLWrr: 7145 case X86::VPMULLWYrr: 7146 case X86::VPMULLWZ128rr: 7147 case X86::VPMULLWZ256rr: 7148 case X86::VPMULLWZrr: 7149 case X86::VPMULLDrr: 7150 case X86::VPMULLDYrr: 7151 case X86::VPMULLDZ128rr: 7152 case X86::VPMULLDZ256rr: 7153 case X86::VPMULLDZrr: 7154 case X86::VPMULLQZ128rr: 7155 case X86::VPMULLQZ256rr: 7156 case X86::VPMULLQZrr: 7157 // Normal min/max instructions are not commutative because of NaN and signed 7158 // zero semantics, but these are. Thus, there's no need to check for global 7159 // relaxed math; the instructions themselves have the properties we need. 7160 case X86::MAXCPDrr: 7161 case X86::MAXCPSrr: 7162 case X86::MAXCSDrr: 7163 case X86::MAXCSSrr: 7164 case X86::MINCPDrr: 7165 case X86::MINCPSrr: 7166 case X86::MINCSDrr: 7167 case X86::MINCSSrr: 7168 case X86::VMAXCPDrr: 7169 case X86::VMAXCPSrr: 7170 case X86::VMAXCPDYrr: 7171 case X86::VMAXCPSYrr: 7172 case X86::VMAXCPDZ128rr: 7173 case X86::VMAXCPSZ128rr: 7174 case X86::VMAXCPDZ256rr: 7175 case X86::VMAXCPSZ256rr: 7176 case X86::VMAXCPDZrr: 7177 case X86::VMAXCPSZrr: 7178 case X86::VMAXCSDrr: 7179 case X86::VMAXCSSrr: 7180 case X86::VMAXCSDZrr: 7181 case X86::VMAXCSSZrr: 7182 case X86::VMINCPDrr: 7183 case X86::VMINCPSrr: 7184 case X86::VMINCPDYrr: 7185 case X86::VMINCPSYrr: 7186 case X86::VMINCPDZ128rr: 7187 case X86::VMINCPSZ128rr: 7188 case X86::VMINCPDZ256rr: 7189 case X86::VMINCPSZ256rr: 7190 case X86::VMINCPDZrr: 7191 case X86::VMINCPSZrr: 7192 case X86::VMINCSDrr: 7193 case X86::VMINCSSrr: 7194 case X86::VMINCSDZrr: 7195 case X86::VMINCSSZrr: 7196 return true; 7197 case X86::ADDPDrr: 7198 case X86::ADDPSrr: 7199 case X86::ADDSDrr: 7200 case X86::ADDSSrr: 7201 case X86::MULPDrr: 7202 case X86::MULPSrr: 7203 case X86::MULSDrr: 7204 case X86::MULSSrr: 7205 case X86::VADDPDrr: 7206 case X86::VADDPSrr: 7207 case X86::VADDPDYrr: 7208 case X86::VADDPSYrr: 7209 case X86::VADDPDZ128rr: 7210 case X86::VADDPSZ128rr: 7211 case X86::VADDPDZ256rr: 7212 case X86::VADDPSZ256rr: 7213 case X86::VADDPDZrr: 7214 case X86::VADDPSZrr: 7215 case X86::VADDSDrr: 7216 case X86::VADDSSrr: 7217 case X86::VADDSDZrr: 7218 case X86::VADDSSZrr: 7219 case X86::VMULPDrr: 7220 case X86::VMULPSrr: 7221 case X86::VMULPDYrr: 7222 case X86::VMULPSYrr: 7223 case X86::VMULPDZ128rr: 7224 case X86::VMULPSZ128rr: 7225 case X86::VMULPDZ256rr: 7226 case X86::VMULPSZ256rr: 7227 case X86::VMULPDZrr: 7228 case X86::VMULPSZrr: 7229 case X86::VMULSDrr: 7230 case X86::VMULSSrr: 7231 case X86::VMULSDZrr: 7232 case X86::VMULSSZrr: 7233 return Inst.getParent()->getParent()->getTarget().Options.UnsafeFPMath; 7234 default: 7235 return false; 7236 } 7237 } 7238 7239 /// This is an architecture-specific helper function of reassociateOps. 7240 /// Set special operand attributes for new instructions after reassociation. 7241 void X86InstrInfo::setSpecialOperandAttr(MachineInstr &OldMI1, 7242 MachineInstr &OldMI2, 7243 MachineInstr &NewMI1, 7244 MachineInstr &NewMI2) const { 7245 // Integer instructions define an implicit EFLAGS source register operand as 7246 // the third source (fourth total) operand. 7247 if (OldMI1.getNumOperands() != 4 || OldMI2.getNumOperands() != 4) 7248 return; 7249 7250 assert(NewMI1.getNumOperands() == 4 && NewMI2.getNumOperands() == 4 && 7251 "Unexpected instruction type for reassociation"); 7252 7253 MachineOperand &OldOp1 = OldMI1.getOperand(3); 7254 MachineOperand &OldOp2 = OldMI2.getOperand(3); 7255 MachineOperand &NewOp1 = NewMI1.getOperand(3); 7256 MachineOperand &NewOp2 = NewMI2.getOperand(3); 7257 7258 assert(OldOp1.isReg() && OldOp1.getReg() == X86::EFLAGS && OldOp1.isDead() && 7259 "Must have dead EFLAGS operand in reassociable instruction"); 7260 assert(OldOp2.isReg() && OldOp2.getReg() == X86::EFLAGS && OldOp2.isDead() && 7261 "Must have dead EFLAGS operand in reassociable instruction"); 7262 7263 (void)OldOp1; 7264 (void)OldOp2; 7265 7266 assert(NewOp1.isReg() && NewOp1.getReg() == X86::EFLAGS && 7267 "Unexpected operand in reassociable instruction"); 7268 assert(NewOp2.isReg() && NewOp2.getReg() == X86::EFLAGS && 7269 "Unexpected operand in reassociable instruction"); 7270 7271 // Mark the new EFLAGS operands as dead to be helpful to subsequent iterations 7272 // of this pass or other passes. The EFLAGS operands must be dead in these new 7273 // instructions because the EFLAGS operands in the original instructions must 7274 // be dead in order for reassociation to occur. 7275 NewOp1.setIsDead(); 7276 NewOp2.setIsDead(); 7277 } 7278 7279 std::pair<unsigned, unsigned> 7280 X86InstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const { 7281 return std::make_pair(TF, 0u); 7282 } 7283 7284 ArrayRef<std::pair<unsigned, const char *>> 7285 X86InstrInfo::getSerializableDirectMachineOperandTargetFlags() const { 7286 using namespace X86II; 7287 static const std::pair<unsigned, const char *> TargetFlags[] = { 7288 {MO_GOT_ABSOLUTE_ADDRESS, "x86-got-absolute-address"}, 7289 {MO_PIC_BASE_OFFSET, "x86-pic-base-offset"}, 7290 {MO_GOT, "x86-got"}, 7291 {MO_GOTOFF, "x86-gotoff"}, 7292 {MO_GOTPCREL, "x86-gotpcrel"}, 7293 {MO_PLT, "x86-plt"}, 7294 {MO_TLSGD, "x86-tlsgd"}, 7295 {MO_TLSLD, "x86-tlsld"}, 7296 {MO_TLSLDM, "x86-tlsldm"}, 7297 {MO_GOTTPOFF, "x86-gottpoff"}, 7298 {MO_INDNTPOFF, "x86-indntpoff"}, 7299 {MO_TPOFF, "x86-tpoff"}, 7300 {MO_DTPOFF, "x86-dtpoff"}, 7301 {MO_NTPOFF, "x86-ntpoff"}, 7302 {MO_GOTNTPOFF, "x86-gotntpoff"}, 7303 {MO_DLLIMPORT, "x86-dllimport"}, 7304 {MO_DARWIN_NONLAZY, "x86-darwin-nonlazy"}, 7305 {MO_DARWIN_NONLAZY_PIC_BASE, "x86-darwin-nonlazy-pic-base"}, 7306 {MO_TLVP, "x86-tlvp"}, 7307 {MO_TLVP_PIC_BASE, "x86-tlvp-pic-base"}, 7308 {MO_SECREL, "x86-secrel"}}; 7309 return makeArrayRef(TargetFlags); 7310 } 7311 7312 namespace { 7313 /// Create Global Base Reg pass. This initializes the PIC 7314 /// global base register for x86-32. 7315 struct CGBR : public MachineFunctionPass { 7316 static char ID; 7317 CGBR() : MachineFunctionPass(ID) {} 7318 7319 bool runOnMachineFunction(MachineFunction &MF) override { 7320 const X86TargetMachine *TM = 7321 static_cast<const X86TargetMachine *>(&MF.getTarget()); 7322 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); 7323 7324 // Don't do anything if this is 64-bit as 64-bit PIC 7325 // uses RIP relative addressing. 7326 if (STI.is64Bit()) 7327 return false; 7328 7329 // Only emit a global base reg in PIC mode. 7330 if (!TM->isPositionIndependent()) 7331 return false; 7332 7333 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); 7334 unsigned GlobalBaseReg = X86FI->getGlobalBaseReg(); 7335 7336 // If we didn't need a GlobalBaseReg, don't insert code. 7337 if (GlobalBaseReg == 0) 7338 return false; 7339 7340 // Insert the set of GlobalBaseReg into the first MBB of the function 7341 MachineBasicBlock &FirstMBB = MF.front(); 7342 MachineBasicBlock::iterator MBBI = FirstMBB.begin(); 7343 DebugLoc DL = FirstMBB.findDebugLoc(MBBI); 7344 MachineRegisterInfo &RegInfo = MF.getRegInfo(); 7345 const X86InstrInfo *TII = STI.getInstrInfo(); 7346 7347 unsigned PC; 7348 if (STI.isPICStyleGOT()) 7349 PC = RegInfo.createVirtualRegister(&X86::GR32RegClass); 7350 else 7351 PC = GlobalBaseReg; 7352 7353 // Operand of MovePCtoStack is completely ignored by asm printer. It's 7354 // only used in JIT code emission as displacement to pc. 7355 BuildMI(FirstMBB, MBBI, DL, TII->get(X86::MOVPC32r), PC).addImm(0); 7356 7357 // If we're using vanilla 'GOT' PIC style, we should use relative addressing 7358 // not to pc, but to _GLOBAL_OFFSET_TABLE_ external. 7359 if (STI.isPICStyleGOT()) { 7360 // Generate addl $__GLOBAL_OFFSET_TABLE_ + [.-piclabel], %some_register 7361 BuildMI(FirstMBB, MBBI, DL, TII->get(X86::ADD32ri), GlobalBaseReg) 7362 .addReg(PC).addExternalSymbol("_GLOBAL_OFFSET_TABLE_", 7363 X86II::MO_GOT_ABSOLUTE_ADDRESS); 7364 } 7365 7366 return true; 7367 } 7368 7369 StringRef getPassName() const override { 7370 return "X86 PIC Global Base Reg Initialization"; 7371 } 7372 7373 void getAnalysisUsage(AnalysisUsage &AU) const override { 7374 AU.setPreservesCFG(); 7375 MachineFunctionPass::getAnalysisUsage(AU); 7376 } 7377 }; 7378 } 7379 7380 char CGBR::ID = 0; 7381 FunctionPass* 7382 llvm::createX86GlobalBaseRegPass() { return new CGBR(); } 7383 7384 namespace { 7385 struct LDTLSCleanup : public MachineFunctionPass { 7386 static char ID; 7387 LDTLSCleanup() : MachineFunctionPass(ID) {} 7388 7389 bool runOnMachineFunction(MachineFunction &MF) override { 7390 if (skipFunction(MF.getFunction())) 7391 return false; 7392 7393 X86MachineFunctionInfo *MFI = MF.getInfo<X86MachineFunctionInfo>(); 7394 if (MFI->getNumLocalDynamicTLSAccesses() < 2) { 7395 // No point folding accesses if there isn't at least two. 7396 return false; 7397 } 7398 7399 MachineDominatorTree *DT = &getAnalysis<MachineDominatorTree>(); 7400 return VisitNode(DT->getRootNode(), 0); 7401 } 7402 7403 // Visit the dominator subtree rooted at Node in pre-order. 7404 // If TLSBaseAddrReg is non-null, then use that to replace any 7405 // TLS_base_addr instructions. Otherwise, create the register 7406 // when the first such instruction is seen, and then use it 7407 // as we encounter more instructions. 7408 bool VisitNode(MachineDomTreeNode *Node, unsigned TLSBaseAddrReg) { 7409 MachineBasicBlock *BB = Node->getBlock(); 7410 bool Changed = false; 7411 7412 // Traverse the current block. 7413 for (MachineBasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; 7414 ++I) { 7415 switch (I->getOpcode()) { 7416 case X86::TLS_base_addr32: 7417 case X86::TLS_base_addr64: 7418 if (TLSBaseAddrReg) 7419 I = ReplaceTLSBaseAddrCall(*I, TLSBaseAddrReg); 7420 else 7421 I = SetRegister(*I, &TLSBaseAddrReg); 7422 Changed = true; 7423 break; 7424 default: 7425 break; 7426 } 7427 } 7428 7429 // Visit the children of this block in the dominator tree. 7430 for (MachineDomTreeNode::iterator I = Node->begin(), E = Node->end(); 7431 I != E; ++I) { 7432 Changed |= VisitNode(*I, TLSBaseAddrReg); 7433 } 7434 7435 return Changed; 7436 } 7437 7438 // Replace the TLS_base_addr instruction I with a copy from 7439 // TLSBaseAddrReg, returning the new instruction. 7440 MachineInstr *ReplaceTLSBaseAddrCall(MachineInstr &I, 7441 unsigned TLSBaseAddrReg) { 7442 MachineFunction *MF = I.getParent()->getParent(); 7443 const X86Subtarget &STI = MF->getSubtarget<X86Subtarget>(); 7444 const bool is64Bit = STI.is64Bit(); 7445 const X86InstrInfo *TII = STI.getInstrInfo(); 7446 7447 // Insert a Copy from TLSBaseAddrReg to RAX/EAX. 7448 MachineInstr *Copy = 7449 BuildMI(*I.getParent(), I, I.getDebugLoc(), 7450 TII->get(TargetOpcode::COPY), is64Bit ? X86::RAX : X86::EAX) 7451 .addReg(TLSBaseAddrReg); 7452 7453 // Erase the TLS_base_addr instruction. 7454 I.eraseFromParent(); 7455 7456 return Copy; 7457 } 7458 7459 // Create a virtual register in *TLSBaseAddrReg, and populate it by 7460 // inserting a copy instruction after I. Returns the new instruction. 7461 MachineInstr *SetRegister(MachineInstr &I, unsigned *TLSBaseAddrReg) { 7462 MachineFunction *MF = I.getParent()->getParent(); 7463 const X86Subtarget &STI = MF->getSubtarget<X86Subtarget>(); 7464 const bool is64Bit = STI.is64Bit(); 7465 const X86InstrInfo *TII = STI.getInstrInfo(); 7466 7467 // Create a virtual register for the TLS base address. 7468 MachineRegisterInfo &RegInfo = MF->getRegInfo(); 7469 *TLSBaseAddrReg = RegInfo.createVirtualRegister(is64Bit 7470 ? &X86::GR64RegClass 7471 : &X86::GR32RegClass); 7472 7473 // Insert a copy from RAX/EAX to TLSBaseAddrReg. 7474 MachineInstr *Next = I.getNextNode(); 7475 MachineInstr *Copy = 7476 BuildMI(*I.getParent(), Next, I.getDebugLoc(), 7477 TII->get(TargetOpcode::COPY), *TLSBaseAddrReg) 7478 .addReg(is64Bit ? X86::RAX : X86::EAX); 7479 7480 return Copy; 7481 } 7482 7483 StringRef getPassName() const override { 7484 return "Local Dynamic TLS Access Clean-up"; 7485 } 7486 7487 void getAnalysisUsage(AnalysisUsage &AU) const override { 7488 AU.setPreservesCFG(); 7489 AU.addRequired<MachineDominatorTree>(); 7490 MachineFunctionPass::getAnalysisUsage(AU); 7491 } 7492 }; 7493 } 7494 7495 char LDTLSCleanup::ID = 0; 7496 FunctionPass* 7497 llvm::createCleanupLocalDynamicTLSPass() { return new LDTLSCleanup(); } 7498 7499 /// Constants defining how certain sequences should be outlined. 7500 /// 7501 /// \p MachineOutlinerDefault implies that the function is called with a call 7502 /// instruction, and a return must be emitted for the outlined function frame. 7503 /// 7504 /// That is, 7505 /// 7506 /// I1 OUTLINED_FUNCTION: 7507 /// I2 --> call OUTLINED_FUNCTION I1 7508 /// I3 I2 7509 /// I3 7510 /// ret 7511 /// 7512 /// * Call construction overhead: 1 (call instruction) 7513 /// * Frame construction overhead: 1 (return instruction) 7514 /// 7515 /// \p MachineOutlinerTailCall implies that the function is being tail called. 7516 /// A jump is emitted instead of a call, and the return is already present in 7517 /// the outlined sequence. That is, 7518 /// 7519 /// I1 OUTLINED_FUNCTION: 7520 /// I2 --> jmp OUTLINED_FUNCTION I1 7521 /// ret I2 7522 /// ret 7523 /// 7524 /// * Call construction overhead: 1 (jump instruction) 7525 /// * Frame construction overhead: 0 (don't need to return) 7526 /// 7527 enum MachineOutlinerClass { 7528 MachineOutlinerDefault, 7529 MachineOutlinerTailCall 7530 }; 7531 7532 outliner::TargetCostInfo 7533 X86InstrInfo::getOutliningCandidateInfo( 7534 std::vector<outliner::Candidate> &RepeatedSequenceLocs) const { 7535 unsigned SequenceSize = std::accumulate( 7536 RepeatedSequenceLocs[0].front(), std::next(RepeatedSequenceLocs[0].back()), 7537 0, [](unsigned Sum, const MachineInstr &MI) { 7538 // FIXME: x86 doesn't implement getInstSizeInBytes, so we can't 7539 // tell the cost. Just assume each instruction is one byte. 7540 if (MI.isDebugInstr() || MI.isKill()) 7541 return Sum; 7542 return Sum + 1; 7543 }); 7544 7545 // FIXME: Use real size in bytes for call and ret instructions. 7546 if (RepeatedSequenceLocs[0].back()->isTerminator()) 7547 return outliner::TargetCostInfo(SequenceSize, 7548 1, // Number of bytes to emit call. 7549 0, // Number of bytes to emit frame. 7550 MachineOutlinerTailCall, // Type of call. 7551 MachineOutlinerTailCall // Type of frame. 7552 ); 7553 7554 return outliner::TargetCostInfo(SequenceSize, 1, 1, MachineOutlinerDefault, 7555 MachineOutlinerDefault); 7556 } 7557 7558 bool X86InstrInfo::isFunctionSafeToOutlineFrom(MachineFunction &MF, 7559 bool OutlineFromLinkOnceODRs) const { 7560 const Function &F = MF.getFunction(); 7561 7562 // Does the function use a red zone? If it does, then we can't risk messing 7563 // with the stack. 7564 if (!F.hasFnAttribute(Attribute::NoRedZone)) { 7565 // It could have a red zone. If it does, then we don't want to touch it. 7566 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); 7567 if (!X86FI || X86FI->getUsesRedZone()) 7568 return false; 7569 } 7570 7571 // If we *don't* want to outline from things that could potentially be deduped 7572 // then return false. 7573 if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage()) 7574 return false; 7575 7576 // This function is viable for outlining, so return true. 7577 return true; 7578 } 7579 7580 outliner::InstrType 7581 X86InstrInfo::getOutliningType(MachineBasicBlock::iterator &MIT, unsigned Flags) const { 7582 MachineInstr &MI = *MIT; 7583 // Don't allow debug values to impact outlining type. 7584 if (MI.isDebugInstr() || MI.isIndirectDebugValue()) 7585 return outliner::InstrType::Invisible; 7586 7587 // At this point, KILL instructions don't really tell us much so we can go 7588 // ahead and skip over them. 7589 if (MI.isKill()) 7590 return outliner::InstrType::Invisible; 7591 7592 // Is this a tail call? If yes, we can outline as a tail call. 7593 if (isTailCall(MI)) 7594 return outliner::InstrType::Legal; 7595 7596 // Is this the terminator of a basic block? 7597 if (MI.isTerminator() || MI.isReturn()) { 7598 7599 // Does its parent have any successors in its MachineFunction? 7600 if (MI.getParent()->succ_empty()) 7601 return outliner::InstrType::Legal; 7602 7603 // It does, so we can't tail call it. 7604 return outliner::InstrType::Illegal; 7605 } 7606 7607 // Don't outline anything that modifies or reads from the stack pointer. 7608 // 7609 // FIXME: There are instructions which are being manually built without 7610 // explicit uses/defs so we also have to check the MCInstrDesc. We should be 7611 // able to remove the extra checks once those are fixed up. For example, 7612 // sometimes we might get something like %rax = POP64r 1. This won't be 7613 // caught by modifiesRegister or readsRegister even though the instruction 7614 // really ought to be formed so that modifiesRegister/readsRegister would 7615 // catch it. 7616 if (MI.modifiesRegister(X86::RSP, &RI) || MI.readsRegister(X86::RSP, &RI) || 7617 MI.getDesc().hasImplicitUseOfPhysReg(X86::RSP) || 7618 MI.getDesc().hasImplicitDefOfPhysReg(X86::RSP)) 7619 return outliner::InstrType::Illegal; 7620 7621 // Outlined calls change the instruction pointer, so don't read from it. 7622 if (MI.readsRegister(X86::RIP, &RI) || 7623 MI.getDesc().hasImplicitUseOfPhysReg(X86::RIP) || 7624 MI.getDesc().hasImplicitDefOfPhysReg(X86::RIP)) 7625 return outliner::InstrType::Illegal; 7626 7627 // Positions can't safely be outlined. 7628 if (MI.isPosition()) 7629 return outliner::InstrType::Illegal; 7630 7631 // Make sure none of the operands of this instruction do anything tricky. 7632 for (const MachineOperand &MOP : MI.operands()) 7633 if (MOP.isCPI() || MOP.isJTI() || MOP.isCFIIndex() || MOP.isFI() || 7634 MOP.isTargetIndex()) 7635 return outliner::InstrType::Illegal; 7636 7637 return outliner::InstrType::Legal; 7638 } 7639 7640 void X86InstrInfo::buildOutlinedFrame(MachineBasicBlock &MBB, 7641 MachineFunction &MF, 7642 const outliner::TargetCostInfo &TCI) 7643 const { 7644 // If we're a tail call, we already have a return, so don't do anything. 7645 if (TCI.FrameConstructionID == MachineOutlinerTailCall) 7646 return; 7647 7648 // We're a normal call, so our sequence doesn't have a return instruction. 7649 // Add it in. 7650 MachineInstr *retq = BuildMI(MF, DebugLoc(), get(X86::RETQ)); 7651 MBB.insert(MBB.end(), retq); 7652 } 7653 7654 MachineBasicBlock::iterator 7655 X86InstrInfo::insertOutlinedCall(Module &M, MachineBasicBlock &MBB, 7656 MachineBasicBlock::iterator &It, 7657 MachineFunction &MF, 7658 const outliner::TargetCostInfo &TCI) const { 7659 // Is it a tail call? 7660 if (TCI.CallConstructionID == MachineOutlinerTailCall) { 7661 // Yes, just insert a JMP. 7662 It = MBB.insert(It, 7663 BuildMI(MF, DebugLoc(), get(X86::JMP_1)) 7664 .addGlobalAddress(M.getNamedValue(MF.getName()))); 7665 } else { 7666 // No, insert a call. 7667 It = MBB.insert(It, 7668 BuildMI(MF, DebugLoc(), get(X86::CALL64pcrel32)) 7669 .addGlobalAddress(M.getNamedValue(MF.getName()))); 7670 } 7671 7672 return It; 7673 } 7674