1 //===- utils/TableGen/X86FoldTablesEmitter.cpp - X86 backend-*- C++ -*-===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This tablegen backend is responsible for emitting the memory fold tables of 11 // the X86 backend instructions. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "CodeGenDAGPatterns.h" 16 #include "CodeGenTarget.h" 17 #include "X86RecognizableInstr.h" 18 #include "llvm/TableGen/Error.h" 19 #include "llvm/TableGen/TableGenBackend.h" 20 21 using namespace llvm; 22 23 namespace { 24 25 // 3 possible strategies for the unfolding flag (TB_NO_REVERSE) of the 26 // manual added entries. 27 enum UnfoldStrategy { 28 UNFOLD, // Allow unfolding 29 NO_UNFOLD, // Prevent unfolding 30 NO_STRATEGY // Make decision according to operands' sizes 31 }; 32 33 // Represents an entry in the manual mapped instructions set. 34 struct ManualMapEntry { 35 const char *RegInstStr; 36 const char *MemInstStr; 37 UnfoldStrategy Strategy; 38 39 ManualMapEntry(const char *RegInstStr, const char *MemInstStr, 40 UnfoldStrategy Strategy = NO_STRATEGY) 41 : RegInstStr(RegInstStr), MemInstStr(MemInstStr), Strategy(Strategy) {} 42 }; 43 44 class IsMatch; 45 46 // List of instructions requiring explicitly aligned memory. 47 const char *const ExplicitAlign[] = {"MOVDQA", "MOVAPS", "MOVAPD", "MOVNTPS", 48 "MOVNTPD", "MOVNTDQ", "MOVNTDQA"}; 49 50 // List of instructions NOT requiring explicit memory alignment. 51 const char *const ExplicitUnalign[] = {"MOVDQU", "MOVUPS", "MOVUPD"}; 52 53 // For manually mapping instructions that do not match by their encoding. 54 const ManualMapEntry ManualMapSet[] = { 55 { "ADD16ri_DB", "ADD16mi", NO_UNFOLD }, 56 { "ADD16ri8_DB", "ADD16mi8", NO_UNFOLD }, 57 { "ADD16rr_DB", "ADD16mr", NO_UNFOLD }, 58 { "ADD32ri_DB", "ADD32mi", NO_UNFOLD }, 59 { "ADD32ri8_DB", "ADD32mi8", NO_UNFOLD }, 60 { "ADD32rr_DB", "ADD32mr", NO_UNFOLD }, 61 { "ADD64ri32_DB", "ADD64mi32", NO_UNFOLD }, 62 { "ADD64ri8_DB", "ADD64mi8", NO_UNFOLD }, 63 { "ADD64rr_DB", "ADD64mr", NO_UNFOLD }, 64 { "ADD16rr_DB", "ADD16rm", NO_UNFOLD }, 65 { "ADD32rr_DB", "ADD32rm", NO_UNFOLD }, 66 { "ADD64rr_DB", "ADD64rm", NO_UNFOLD }, 67 { "PUSH16r", "PUSH16rmm", NO_UNFOLD }, 68 { "PUSH32r", "PUSH32rmm", NO_UNFOLD }, 69 { "PUSH64r", "PUSH64rmm", NO_UNFOLD }, 70 { "TAILJMPr", "TAILJMPm", UNFOLD }, 71 { "TAILJMPr64", "TAILJMPm64", UNFOLD }, 72 { "TAILJMPr64_REX", "TAILJMPm64_REX", UNFOLD }, 73 }; 74 75 // Do not add these instructions to any of the folding tables. 76 const char *const NoFoldSet[] = { 77 "TCRETURNri64", 78 "TCRETURNmi64", // Special dealing (in X86InstrCompiler.td under 79 "TCRETURNri", // "tailcall stuff" section). 80 "TCRETURNmi" 81 82 // Different calculations of the folded operand between 83 // memory and register forms (folding is illegal). 84 // - In their register form, the second register operand's relevant 85 // bits are only the first 4/5/6 (depending on mode and reg size). 86 // - In their memory form, the second register operand's relevant 87 // bits are only the first 16/32/64 (depending on mode and reg size). 88 "BT16rr", "BT32rr", "BT64rr", 89 "BT16mr", "BT32mr", "BT64mr", 90 "BTC16rr", "BTC32rr", "BTC64rr", 91 "BTC16mr", "BTC32mr", "BTC64mr", 92 "BTR16rr", "BTR32rr", "BTR64rr", 93 "BTR16mr", "BTR32mr", "BTR64mr", 94 "BTS16rr", "BTS32rr", "BTS64rr", 95 "BTS16mr", "BTS32mr", "BTS64mr", 96 97 // Memory folding is enabled only when optimizing for size by DAG 98 // patterns only. (issue detailed in D28744 review) 99 "VCVTSS2SDrm", "VCVTSS2SDrr", 100 "VCVTSS2SDZrm", "VCVTSS2SDZrr", 101 "VCVTSS2SDZrmk", "VCVTSS2SDZrrk", 102 "VCVTSS2SDZrmkz", "VCVTSS2SDZrrkz", 103 "VCVTSS2SDZrm_Int", "VCVTSS2SDZrr_Int", 104 "VCVTSS2SDZrm_Intk", "VCVTSS2SDZrr_Intk", 105 "VCVTSS2SDZrm_Intkz", "VCVTSS2SDZrr_Intkz", 106 "VCVTSD2SSrm", "VCVTSD2SSrr", 107 "VCVTSD2SSZrm", "VCVTSD2SSZrr", 108 "VCVTSD2SSZrmk", "VCVTSD2SSZrrk", 109 "VCVTSD2SSZrmkz", "VCVTSD2SSZrrkz", 110 "VCVTSD2SSZrm_Int", "VCVTSD2SSZrr_Int", 111 "VCVTSD2SSZrm_Intk", "VCVTSD2SSZrr_Intk", 112 "VCVTSD2SSZrm_Intkz", "VCVTSD2SSZrr_Intkz", 113 "VRCP14SSrm", "VRCP14SSrr", 114 "VRCP14SDrm", "VRCP14SDrr", 115 "VRSQRT14SSrm", "VRSQRT14SSrr", 116 "VRSQRT14SDrm", "VRSQRT14SDrr", 117 "VSQRTSSm", "VSQRTSSr", 118 "VSQRTSSm_Int", "VSQRTSSr_Int", 119 "VSQRTSSZm", "VSQRTSSZr", 120 "VSQRTSSZm_Int", "VSQRTSSZr_Int", 121 "VSQRTSSZm_Intk", "VSQRTSSZr_Intk", 122 "VSQRTSSZm_Intkz", "VSQRTSSZr_Intkz", 123 "VSQRTSDm", "VSQRTSDr", 124 "VSQRTSDm_Int", "VSQRTSDr_Int", 125 "VSQRTSDZm", "VSQRTSDZr", 126 "VSQRTSDZm_Int", "VSQRTSDZr_Int", 127 "VSQRTSDZm_Intk", "VSQRTSDZr_Intk", 128 "VSQRTSDZm_Intkz", "VSQRTSDZr_Intkz", 129 }; 130 131 static bool isExplicitAlign(const CodeGenInstruction *Inst) { 132 return any_of(ExplicitAlign, [Inst](const char *InstStr) { 133 return Inst->TheDef->getName().find(InstStr) != StringRef::npos; 134 }); 135 } 136 137 static bool isExplicitUnalign(const CodeGenInstruction *Inst) { 138 return any_of(ExplicitUnalign, [Inst](const char *InstStr) { 139 return Inst->TheDef->getName().find(InstStr) != StringRef::npos; 140 }); 141 } 142 143 class X86FoldTablesEmitter { 144 RecordKeeper &Records; 145 CodeGenTarget Target; 146 147 // Represents an entry in the folding table 148 class X86FoldTableEntry { 149 const CodeGenInstruction *RegInst; 150 const CodeGenInstruction *MemInst; 151 152 public: 153 bool CannotUnfold = false; 154 bool IsLoad = false; 155 bool IsStore = false; 156 bool IsAligned = false; 157 unsigned int Alignment = 0; 158 159 X86FoldTableEntry(const CodeGenInstruction *RegInst, 160 const CodeGenInstruction *MemInst) 161 : RegInst(RegInst), MemInst(MemInst) {} 162 163 friend raw_ostream &operator<<(raw_ostream &OS, 164 const X86FoldTableEntry &E) { 165 OS << "{ X86::" << E.RegInst->TheDef->getName() 166 << ", X86::" << E.MemInst->TheDef->getName() << ", "; 167 168 if (E.IsLoad) 169 OS << "TB_FOLDED_LOAD | "; 170 if (E.IsStore) 171 OS << "TB_FOLDED_STORE | "; 172 if (E.CannotUnfold) 173 OS << "TB_NO_REVERSE | "; 174 if (E.IsAligned) 175 OS << "TB_ALIGN_" << E.Alignment << " | "; 176 177 OS << "0 },\n"; 178 179 return OS; 180 } 181 }; 182 183 typedef std::vector<X86FoldTableEntry> FoldTable; 184 // std::vector for each folding table. 185 // Table2Addr - Holds instructions which their memory form performs load+store 186 // Table#i - Holds instructions which the their memory form perform a load OR 187 // a store, and their #i'th operand is folded. 188 FoldTable Table2Addr; 189 FoldTable Table0; 190 FoldTable Table1; 191 FoldTable Table2; 192 FoldTable Table3; 193 FoldTable Table4; 194 195 public: 196 X86FoldTablesEmitter(RecordKeeper &R) : Records(R), Target(R) {} 197 198 // run - Generate the 6 X86 memory fold tables. 199 void run(raw_ostream &OS); 200 201 private: 202 // Decides to which table to add the entry with the given instructions. 203 // S sets the strategy of adding the TB_NO_REVERSE flag. 204 void updateTables(const CodeGenInstruction *RegInstr, 205 const CodeGenInstruction *MemInstr, 206 const UnfoldStrategy S = NO_STRATEGY); 207 208 // Generates X86FoldTableEntry with the given instructions and fill it with 209 // the appropriate flags - then adds it to Table. 210 void addEntryWithFlags(FoldTable &Table, const CodeGenInstruction *RegInstr, 211 const CodeGenInstruction *MemInstr, 212 const UnfoldStrategy S, const unsigned int FoldedInd); 213 214 // Print the given table as a static const C++ array of type 215 // X86MemoryFoldTableEntry. 216 void printTable(const FoldTable &Table, std::string TableName, 217 raw_ostream &OS) { 218 OS << "\nstatic const X86MemoryFoldTableEntry MemoryFold" << TableName 219 << "[] = {\n"; 220 221 for (const X86FoldTableEntry &E : Table) 222 OS.indent(2) << E; 223 224 OS << "};\n"; 225 } 226 }; 227 228 // Return true if one of the instruction's operands is a RST register class 229 static bool hasRSTRegClass(const CodeGenInstruction *Inst) { 230 return any_of(Inst->Operands, [](const CGIOperandList::OperandInfo &OpIn) { 231 return OpIn.Rec->getName() == "RST"; 232 }); 233 } 234 235 // Return true if one of the instruction's operands is a ptr_rc_tailcall 236 static bool hasPtrTailcallRegClass(const CodeGenInstruction *Inst) { 237 return any_of(Inst->Operands, [](const CGIOperandList::OperandInfo &OpIn) { 238 return OpIn.Rec->getName() == "ptr_rc_tailcall"; 239 }); 240 } 241 242 // Calculates the integer value representing the BitsInit object 243 static inline uint64_t getValueFromBitsInit(const BitsInit *B) { 244 assert(B->getNumBits() <= sizeof(uint64_t) * CHAR_BIT && 245 "BitInits' too long!"); 246 247 uint64_t Value = 0; 248 for (unsigned i = 0, e = B->getNumBits(); i != e; ++i) { 249 BitInit *Bit = cast<BitInit>(B->getBit(i)); 250 Value |= uint64_t(Bit->getValue()) << i; 251 } 252 return Value; 253 } 254 255 // Returns true if the two given BitsInits represent the same integer value 256 static inline bool equalBitsInits(const BitsInit *B1, const BitsInit *B2) { 257 if (B1->getNumBits() != B2->getNumBits()) 258 PrintFatalError("Comparing two BitsInits with different sizes!"); 259 260 for (unsigned i = 0, e = B1->getNumBits(); i != e; ++i) { 261 BitInit *Bit1 = cast<BitInit>(B1->getBit(i)); 262 BitInit *Bit2 = cast<BitInit>(B2->getBit(i)); 263 if (Bit1->getValue() != Bit2->getValue()) 264 return false; 265 } 266 return true; 267 } 268 269 // Return the size of the register operand 270 static inline unsigned int getRegOperandSize(const Record *RegRec) { 271 if (RegRec->isSubClassOf("RegisterOperand")) 272 RegRec = RegRec->getValueAsDef("RegClass"); 273 if (RegRec->isSubClassOf("RegisterClass")) 274 return RegRec->getValueAsListOfDefs("RegTypes")[0]->getValueAsInt("Size"); 275 276 llvm_unreachable("Register operand's size not known!"); 277 } 278 279 // Return the size of the memory operand 280 static inline unsigned int 281 getMemOperandSize(const Record *MemRec, const bool IntrinsicSensitive = false) { 282 if (MemRec->isSubClassOf("Operand")) { 283 // Intrinsic memory instructions use ssmem/sdmem. 284 if (IntrinsicSensitive && 285 (MemRec->getName() == "sdmem" || MemRec->getName() == "ssmem")) 286 return 128; 287 288 StringRef Name = 289 MemRec->getValueAsDef("ParserMatchClass")->getValueAsString("Name"); 290 if (Name == "Mem8") 291 return 8; 292 if (Name == "Mem16") 293 return 16; 294 if (Name == "Mem32") 295 return 32; 296 if (Name == "Mem64") 297 return 64; 298 if (Name == "Mem80") 299 return 80; 300 if (Name == "Mem128") 301 return 128; 302 if (Name == "Mem256") 303 return 256; 304 if (Name == "Mem512") 305 return 512; 306 } 307 308 llvm_unreachable("Memory operand's size not known!"); 309 } 310 311 // Returns true if the record's list of defs includes the given def. 312 static inline bool hasDefInList(const Record *Rec, const StringRef List, 313 const StringRef Def) { 314 if (!Rec->isValueUnset(List)) { 315 return any_of(*(Rec->getValueAsListInit(List)), 316 [Def](const Init *I) { return I->getAsString() == Def; }); 317 } 318 return false; 319 } 320 321 // Return true if the instruction defined as a register flavor. 322 static inline bool hasRegisterFormat(const Record *Inst) { 323 const BitsInit *FormBits = Inst->getValueAsBitsInit("FormBits"); 324 uint64_t FormBitsNum = getValueFromBitsInit(FormBits); 325 326 // Values from X86Local namespace defined in X86RecognizableInstr.cpp 327 return FormBitsNum >= X86Local::MRMDestReg && FormBitsNum <= X86Local::MRM7r; 328 } 329 330 // Return true if the instruction defined as a memory flavor. 331 static inline bool hasMemoryFormat(const Record *Inst) { 332 const BitsInit *FormBits = Inst->getValueAsBitsInit("FormBits"); 333 uint64_t FormBitsNum = getValueFromBitsInit(FormBits); 334 335 // Values from X86Local namespace defined in X86RecognizableInstr.cpp 336 return FormBitsNum >= X86Local::MRMDestMem && FormBitsNum <= X86Local::MRM7m; 337 } 338 339 static inline bool isNOREXRegClass(const Record *Op) { 340 return Op->getName().find("_NOREX") != StringRef::npos; 341 } 342 343 static inline bool isRegisterOperand(const Record *Rec) { 344 return Rec->isSubClassOf("RegisterClass") || 345 Rec->isSubClassOf("RegisterOperand") || 346 Rec->isSubClassOf("PointerLikeRegClass"); 347 } 348 349 static inline bool isMemoryOperand(const Record *Rec) { 350 return Rec->isSubClassOf("Operand") && 351 Rec->getValueAsString("OperandType") == "OPERAND_MEMORY"; 352 } 353 354 static inline bool isImmediateOperand(const Record *Rec) { 355 return Rec->isSubClassOf("Operand") && 356 Rec->getValueAsString("OperandType") == "OPERAND_IMMEDIATE"; 357 } 358 359 // Get the alternative instruction pointed by "FoldGenRegForm" field. 360 static inline const CodeGenInstruction * 361 getAltRegInst(const CodeGenInstruction *I, const RecordKeeper &Records, 362 const CodeGenTarget &Target) { 363 364 std::string AltRegInstStr = I->TheDef->getValueAsString("FoldGenRegForm"); 365 Record *AltRegInstRec = Records.getDef(AltRegInstStr); 366 assert(AltRegInstRec && 367 "Alternative register form instruction def not found"); 368 CodeGenInstruction &AltRegInst = Target.getInstruction(AltRegInstRec); 369 return &AltRegInst; 370 } 371 372 // Function object - Operator() returns true if the given VEX instruction 373 // matches the EVEX instruction of this object. 374 class IsMatch { 375 const CodeGenInstruction *MemInst; 376 const RecordKeeper &Records; 377 378 public: 379 IsMatch(const CodeGenInstruction *Inst, const RecordKeeper &Records) 380 : MemInst(Inst), Records(Records) {} 381 382 bool operator()(const CodeGenInstruction *RegInst) { 383 Record *MemRec = MemInst->TheDef; 384 Record *RegRec = RegInst->TheDef; 385 386 // Return false if one (at least) of the encoding fields of both 387 // instructions do not match. 388 if (RegRec->getValueAsDef("OpEnc") != MemRec->getValueAsDef("OpEnc") || 389 !equalBitsInits(RegRec->getValueAsBitsInit("Opcode"), 390 MemRec->getValueAsBitsInit("Opcode")) || 391 // VEX/EVEX fields 392 RegRec->getValueAsDef("OpPrefix") != 393 MemRec->getValueAsDef("OpPrefix") || 394 RegRec->getValueAsDef("OpMap") != MemRec->getValueAsDef("OpMap") || 395 RegRec->getValueAsDef("OpSize") != MemRec->getValueAsDef("OpSize") || 396 RegRec->getValueAsBit("hasVEX_4V") != 397 MemRec->getValueAsBit("hasVEX_4V") || 398 RegRec->getValueAsBit("hasEVEX_K") != 399 MemRec->getValueAsBit("hasEVEX_K") || 400 RegRec->getValueAsBit("hasEVEX_Z") != 401 MemRec->getValueAsBit("hasEVEX_Z") || 402 RegRec->getValueAsBit("hasEVEX_B") != 403 MemRec->getValueAsBit("hasEVEX_B") || 404 RegRec->getValueAsBit("hasEVEX_RC") != 405 MemRec->getValueAsBit("hasEVEX_RC") || 406 RegRec->getValueAsBit("hasREX_WPrefix") != 407 MemRec->getValueAsBit("hasREX_WPrefix") || 408 RegRec->getValueAsBit("hasLockPrefix") != 409 MemRec->getValueAsBit("hasLockPrefix") || 410 !equalBitsInits(RegRec->getValueAsBitsInit("EVEX_LL"), 411 MemRec->getValueAsBitsInit("EVEX_LL")) || 412 !equalBitsInits(RegRec->getValueAsBitsInit("VEX_WPrefix"), 413 MemRec->getValueAsBitsInit("VEX_WPrefix")) || 414 // Instruction's format - The register form's "Form" field should be 415 // the opposite of the memory form's "Form" field. 416 !areOppositeForms(RegRec->getValueAsBitsInit("FormBits"), 417 MemRec->getValueAsBitsInit("FormBits")) || 418 RegRec->getValueAsBit("isAsmParserOnly") != 419 MemRec->getValueAsBit("isAsmParserOnly")) 420 return false; 421 422 // Make sure the sizes of the operands of both instructions suit each other. 423 // This is needed for instructions with intrinsic version (_Int). 424 // Where the only difference is the size of the operands. 425 // For example: VUCOMISDZrm and Int_VUCOMISDrm 426 // Also for instructions that their EVEX version was upgraded to work with 427 // k-registers. For example VPCMPEQBrm (xmm output register) and 428 // VPCMPEQBZ128rm (k register output register). 429 bool ArgFolded = false; 430 unsigned MemOutSize = MemRec->getValueAsDag("OutOperandList")->getNumArgs(); 431 unsigned RegOutSize = RegRec->getValueAsDag("OutOperandList")->getNumArgs(); 432 unsigned MemInSize = MemRec->getValueAsDag("InOperandList")->getNumArgs(); 433 unsigned RegInSize = RegRec->getValueAsDag("InOperandList")->getNumArgs(); 434 435 // Instructions with one output in their memory form use the memory folded 436 // operand as source and destination (Read-Modify-Write). 437 unsigned RegStartIdx = 438 (MemOutSize + 1 == RegOutSize) && (MemInSize == RegInSize) ? 1 : 0; 439 440 for (unsigned i = 0, e = MemInst->Operands.size(); i < e; i++) { 441 Record *MemOpRec = MemInst->Operands[i].Rec; 442 Record *RegOpRec = RegInst->Operands[i + RegStartIdx].Rec; 443 444 if (MemOpRec == RegOpRec) 445 continue; 446 447 if (isRegisterOperand(MemOpRec) && isRegisterOperand(RegOpRec)) { 448 if (getRegOperandSize(MemOpRec) != getRegOperandSize(RegOpRec) || 449 isNOREXRegClass(MemOpRec) != isNOREXRegClass(RegOpRec)) 450 return false; 451 } else if (isMemoryOperand(MemOpRec) && isMemoryOperand(RegOpRec)) { 452 if (getMemOperandSize(MemOpRec) != getMemOperandSize(RegOpRec)) 453 return false; 454 } else if (isImmediateOperand(MemOpRec) && isImmediateOperand(RegOpRec)) { 455 if (MemOpRec->getValueAsDef("Type") != RegOpRec->getValueAsDef("Type")) 456 return false; 457 } else { 458 // Only one operand can be folded. 459 if (ArgFolded) 460 return false; 461 462 assert(isRegisterOperand(RegOpRec) && isMemoryOperand(MemOpRec)); 463 ArgFolded = true; 464 } 465 } 466 467 return true; 468 } 469 470 private: 471 // Return true of the 2 given forms are the opposite of each other. 472 bool areOppositeForms(const BitsInit *RegFormBits, 473 const BitsInit *MemFormBits) { 474 uint64_t MemFormNum = getValueFromBitsInit(MemFormBits); 475 uint64_t RegFormNum = getValueFromBitsInit(RegFormBits); 476 477 if ((MemFormNum == X86Local::MRM0m && RegFormNum == X86Local::MRM0r) || 478 (MemFormNum == X86Local::MRM1m && RegFormNum == X86Local::MRM1r) || 479 (MemFormNum == X86Local::MRM2m && RegFormNum == X86Local::MRM2r) || 480 (MemFormNum == X86Local::MRM3m && RegFormNum == X86Local::MRM3r) || 481 (MemFormNum == X86Local::MRM4m && RegFormNum == X86Local::MRM4r) || 482 (MemFormNum == X86Local::MRM5m && RegFormNum == X86Local::MRM5r) || 483 (MemFormNum == X86Local::MRM6m && RegFormNum == X86Local::MRM6r) || 484 (MemFormNum == X86Local::MRM7m && RegFormNum == X86Local::MRM7r) || 485 (MemFormNum == X86Local::MRMXm && RegFormNum == X86Local::MRMXr) || 486 (MemFormNum == X86Local::MRMDestMem && 487 RegFormNum == X86Local::MRMDestReg) || 488 (MemFormNum == X86Local::MRMSrcMem && 489 RegFormNum == X86Local::MRMSrcReg) || 490 (MemFormNum == X86Local::MRMSrcMem4VOp3 && 491 RegFormNum == X86Local::MRMSrcReg4VOp3) || 492 (MemFormNum == X86Local::MRMSrcMemOp4 && 493 RegFormNum == X86Local::MRMSrcRegOp4)) 494 return true; 495 496 return false; 497 } 498 }; 499 500 } // end anonymous namespace 501 502 void X86FoldTablesEmitter::addEntryWithFlags(FoldTable &Table, 503 const CodeGenInstruction *RegInstr, 504 const CodeGenInstruction *MemInstr, 505 const UnfoldStrategy S, 506 const unsigned int FoldedInd) { 507 508 X86FoldTableEntry Result = X86FoldTableEntry(RegInstr, MemInstr); 509 Record *RegRec = RegInstr->TheDef; 510 Record *MemRec = MemInstr->TheDef; 511 512 // Only table0 entries should explicitly specify a load or store flag. 513 if (&Table == &Table0) { 514 unsigned MemInOpsNum = MemRec->getValueAsDag("InOperandList")->getNumArgs(); 515 unsigned RegInOpsNum = RegRec->getValueAsDag("InOperandList")->getNumArgs(); 516 // If the instruction writes to the folded operand, it will appear as an 517 // output in the register form instruction and as an input in the memory 518 // form instruction. 519 // If the instruction reads from the folded operand, it well appear as in 520 // input in both forms. 521 if (MemInOpsNum == RegInOpsNum) 522 Result.IsLoad = true; 523 else 524 Result.IsStore = true; 525 } 526 527 Record *RegOpRec = RegInstr->Operands[FoldedInd].Rec; 528 Record *MemOpRec = MemInstr->Operands[FoldedInd].Rec; 529 530 // Unfolding code generates a load/store instruction according to the size of 531 // the register in the register form instruction. 532 // If the register's size is greater than the memory's operand size, do not 533 // allow unfolding. 534 if (S == UNFOLD) 535 Result.CannotUnfold = false; 536 else if (S == NO_UNFOLD) 537 Result.CannotUnfold = true; 538 else if (getRegOperandSize(RegOpRec) > getMemOperandSize(MemOpRec)) 539 Result.CannotUnfold = true; // S == NO_STRATEGY 540 541 uint64_t Enc = getValueFromBitsInit(RegRec->getValueAsBitsInit("OpEncBits")); 542 if (isExplicitAlign(RegInstr)) { 543 // The instruction require explicitly aligned memory. 544 BitsInit *VectSize = RegRec->getValueAsBitsInit("VectSize"); 545 uint64_t Value = getValueFromBitsInit(VectSize); 546 Result.IsAligned = true; 547 Result.Alignment = Value; 548 } else if (Enc != X86Local::XOP && Enc != X86Local::VEX && 549 Enc != X86Local::EVEX) { 550 // Instructions with VEX encoding do not require alignment. 551 if (!isExplicitUnalign(RegInstr) && getMemOperandSize(MemOpRec) > 64) { 552 // SSE packed vector instructions require a 16 byte alignment. 553 Result.IsAligned = true; 554 Result.Alignment = 16; 555 } 556 } 557 558 Table.push_back(Result); 559 } 560 561 void X86FoldTablesEmitter::updateTables(const CodeGenInstruction *RegInstr, 562 const CodeGenInstruction *MemInstr, 563 const UnfoldStrategy S) { 564 565 Record *RegRec = RegInstr->TheDef; 566 Record *MemRec = MemInstr->TheDef; 567 unsigned MemOutSize = MemRec->getValueAsDag("OutOperandList")->getNumArgs(); 568 unsigned RegOutSize = RegRec->getValueAsDag("OutOperandList")->getNumArgs(); 569 unsigned MemInSize = MemRec->getValueAsDag("InOperandList")->getNumArgs(); 570 unsigned RegInSize = RegRec->getValueAsDag("InOperandList")->getNumArgs(); 571 572 // Instructions which have the WriteRMW value (Read-Modify-Write) should be 573 // added to Table2Addr. 574 if (hasDefInList(MemRec, "SchedRW", "WriteRMW") && MemOutSize != RegOutSize && 575 MemInSize == RegInSize) { 576 addEntryWithFlags(Table2Addr, RegInstr, MemInstr, S, 0); 577 return; 578 } 579 580 if (MemInSize == RegInSize && MemOutSize == RegOutSize) { 581 // Load-Folding cases. 582 // If the i'th register form operand is a register and the i'th memory form 583 // operand is a memory operand, add instructions to Table#i. 584 for (unsigned i = RegOutSize, e = RegInstr->Operands.size(); i < e; i++) { 585 Record *RegOpRec = RegInstr->Operands[i].Rec; 586 Record *MemOpRec = MemInstr->Operands[i].Rec; 587 if (isRegisterOperand(RegOpRec) && isMemoryOperand(MemOpRec)) { 588 switch (i) { 589 default: llvm_unreachable("Unexpected operand count!"); 590 case 0: 591 addEntryWithFlags(Table0, RegInstr, MemInstr, S, 0); 592 return; 593 case 1: 594 addEntryWithFlags(Table1, RegInstr, MemInstr, S, 1); 595 return; 596 case 2: 597 addEntryWithFlags(Table2, RegInstr, MemInstr, S, 2); 598 return; 599 case 3: 600 addEntryWithFlags(Table3, RegInstr, MemInstr, S, 3); 601 return; 602 case 4: 603 addEntryWithFlags(Table4, RegInstr, MemInstr, S, 4); 604 return; 605 } 606 } 607 } 608 } else if (MemInSize == RegInSize + 1 && MemOutSize + 1 == RegOutSize) { 609 // Store-Folding cases. 610 // If the memory form instruction performs performs a store, the *output* 611 // register of the register form instructions disappear and instead a 612 // memory *input* operand appears in the memory form instruction. 613 // For example: 614 // MOVAPSrr => (outs VR128:$dst), (ins VR128:$src) 615 // MOVAPSmr => (outs), (ins f128mem:$dst, VR128:$src) 616 Record *RegOpRec = RegInstr->Operands[RegOutSize - 1].Rec; 617 Record *MemOpRec = MemInstr->Operands[RegOutSize - 1].Rec; 618 if (isRegisterOperand(RegOpRec) && isMemoryOperand(MemOpRec)) 619 addEntryWithFlags(Table0, RegInstr, MemInstr, S, 0); 620 } 621 622 return; 623 } 624 625 void X86FoldTablesEmitter::run(raw_ostream &OS) { 626 emitSourceFileHeader("X86 fold tables", OS); 627 628 // Holds all memory instructions 629 std::vector<const CodeGenInstruction *> MemInsts; 630 // Holds all register instructions - divided according to opcode. 631 std::map<uint8_t, std::vector<const CodeGenInstruction *>> RegInsts; 632 633 ArrayRef<const CodeGenInstruction *> NumberedInstructions = 634 Target.getInstructionsByEnumValue(); 635 636 for (const CodeGenInstruction *Inst : NumberedInstructions) { 637 if (!Inst->TheDef->getNameInit() || !Inst->TheDef->isSubClassOf("X86Inst")) 638 continue; 639 640 const Record *Rec = Inst->TheDef; 641 642 // - Do not proceed matching if the instruction in NoFoldSet. 643 // - Instructions including RST register class operands are not relevant 644 // for memory folding (for further details check the explanation in 645 // lib/Target/X86/X86InstrFPStack.td file). 646 // - Some instructions (listed in the manual map above) use the register 647 // class ptr_rc_tailcall, which can be of a size 32 or 64, to ensure 648 // safe mapping of these instruction we manually map them and exclude 649 // them from the automation. 650 if (find(NoFoldSet, Rec->getName()) != std::end(NoFoldSet) || 651 hasRSTRegClass(Inst) || hasPtrTailcallRegClass(Inst)) 652 continue; 653 654 // Add all the memory form instructions to MemInsts, and all the register 655 // form instructions to RegInsts[Opc], where Opc in the opcode of each 656 // instructions. this helps reducing the runtime of the backend. 657 if (hasMemoryFormat(Rec)) 658 MemInsts.push_back(Inst); 659 else if (hasRegisterFormat(Rec)) { 660 uint8_t Opc = getValueFromBitsInit(Rec->getValueAsBitsInit("Opcode")); 661 RegInsts[Opc].push_back(Inst); 662 } 663 } 664 665 // For each memory form instruction, try to find its register form 666 // instruction. 667 for (const CodeGenInstruction *MemInst : MemInsts) { 668 uint8_t Opc = 669 getValueFromBitsInit(MemInst->TheDef->getValueAsBitsInit("Opcode")); 670 671 if (RegInsts.count(Opc) == 0) 672 continue; 673 674 // Two forms (memory & register) of the same instruction must have the same 675 // opcode. try matching only with register form instructions with the same 676 // opcode. 677 std::vector<const CodeGenInstruction *> &OpcRegInsts = 678 RegInsts.find(Opc)->second; 679 680 auto Match = find_if(OpcRegInsts, IsMatch(MemInst, Records)); 681 if (Match != OpcRegInsts.end()) { 682 const CodeGenInstruction *RegInst = *Match; 683 // If the matched instruction has it's "FoldGenRegForm" set, map the 684 // memory form instruction to the register form instruction pointed by 685 // this field 686 if (RegInst->TheDef->isValueUnset("FoldGenRegForm")) { 687 updateTables(RegInst, MemInst); 688 } else { 689 const CodeGenInstruction *AltRegInst = 690 getAltRegInst(RegInst, Records, Target); 691 updateTables(AltRegInst, MemInst); 692 } 693 OpcRegInsts.erase(Match); 694 } 695 } 696 697 // Add the manually mapped instructions listed above. 698 for (const ManualMapEntry &Entry : ManualMapSet) { 699 Record *RegInstIter = Records.getDef(Entry.RegInstStr); 700 Record *MemInstIter = Records.getDef(Entry.MemInstStr); 701 702 updateTables(&(Target.getInstruction(RegInstIter)), 703 &(Target.getInstruction(MemInstIter)), Entry.Strategy); 704 } 705 706 // Print all tables to raw_ostream OS. 707 printTable(Table2Addr, "Table2Addr", OS); 708 printTable(Table0, "Table0", OS); 709 printTable(Table1, "Table1", OS); 710 printTable(Table2, "Table2", OS); 711 printTable(Table3, "Table3", OS); 712 printTable(Table4, "Table4", OS); 713 } 714 715 namespace llvm { 716 717 void EmitX86FoldTables(RecordKeeper &RK, raw_ostream &OS) { 718 X86FoldTablesEmitter(RK).run(OS); 719 } 720 } // namespace llvm 721