1 //===-- Target.cpp ----------------------------------------------*- C++ -*-===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 #include "../Target.h"
10 
11 #include "../Latency.h"
12 #include "../Uops.h"
13 #include "MCTargetDesc/X86BaseInfo.h"
14 #include "MCTargetDesc/X86MCTargetDesc.h"
15 #include "X86.h"
16 #include "X86RegisterInfo.h"
17 #include "X86Subtarget.h"
18 #include "llvm/MC/MCInstBuilder.h"
19 
20 namespace llvm {
21 namespace exegesis {
22 
23 // Returns an error if we cannot handle the memory references in this
24 // instruction.
25 static Error isInvalidMemoryInstr(const Instruction &Instr) {
26   switch (Instr.Description->TSFlags & X86II::FormMask) {
27   default:
28     llvm_unreachable("Unknown FormMask value");
29   // These have no memory access.
30   case X86II::Pseudo:
31   case X86II::RawFrm:
32   case X86II::MRMDestReg:
33   case X86II::MRMSrcReg:
34   case X86II::MRMSrcReg4VOp3:
35   case X86II::MRMSrcRegOp4:
36   case X86II::MRMXr:
37   case X86II::MRM0r:
38   case X86II::MRM1r:
39   case X86II::MRM2r:
40   case X86II::MRM3r:
41   case X86II::MRM4r:
42   case X86II::MRM5r:
43   case X86II::MRM6r:
44   case X86II::MRM7r:
45   case X86II::MRM_C0:
46   case X86II::MRM_C1:
47   case X86II::MRM_C2:
48   case X86II::MRM_C3:
49   case X86II::MRM_C4:
50   case X86II::MRM_C5:
51   case X86II::MRM_C6:
52   case X86II::MRM_C7:
53   case X86II::MRM_C8:
54   case X86II::MRM_C9:
55   case X86II::MRM_CA:
56   case X86II::MRM_CB:
57   case X86II::MRM_CC:
58   case X86II::MRM_CD:
59   case X86II::MRM_CE:
60   case X86II::MRM_CF:
61   case X86II::MRM_D0:
62   case X86II::MRM_D1:
63   case X86II::MRM_D2:
64   case X86II::MRM_D3:
65   case X86II::MRM_D4:
66   case X86II::MRM_D5:
67   case X86II::MRM_D6:
68   case X86II::MRM_D7:
69   case X86II::MRM_D8:
70   case X86II::MRM_D9:
71   case X86II::MRM_DA:
72   case X86II::MRM_DB:
73   case X86II::MRM_DC:
74   case X86II::MRM_DD:
75   case X86II::MRM_DE:
76   case X86II::MRM_DF:
77   case X86II::MRM_E0:
78   case X86II::MRM_E1:
79   case X86II::MRM_E2:
80   case X86II::MRM_E3:
81   case X86II::MRM_E4:
82   case X86II::MRM_E5:
83   case X86II::MRM_E6:
84   case X86II::MRM_E7:
85   case X86II::MRM_E8:
86   case X86II::MRM_E9:
87   case X86II::MRM_EA:
88   case X86II::MRM_EB:
89   case X86II::MRM_EC:
90   case X86II::MRM_ED:
91   case X86II::MRM_EE:
92   case X86II::MRM_EF:
93   case X86II::MRM_F0:
94   case X86II::MRM_F1:
95   case X86II::MRM_F2:
96   case X86II::MRM_F3:
97   case X86II::MRM_F4:
98   case X86II::MRM_F5:
99   case X86II::MRM_F6:
100   case X86II::MRM_F7:
101   case X86II::MRM_F8:
102   case X86II::MRM_F9:
103   case X86II::MRM_FA:
104   case X86II::MRM_FB:
105   case X86II::MRM_FC:
106   case X86II::MRM_FD:
107   case X86II::MRM_FE:
108   case X86II::MRM_FF:
109   case X86II::RawFrmImm8:
110     return Error::success();
111   case X86II::AddRegFrm:
112     return (Instr.Description->Opcode == X86::POP16r || Instr.Description->Opcode == X86::POP32r ||
113             Instr.Description->Opcode == X86::PUSH16r || Instr.Description->Opcode == X86::PUSH32r)
114                ? make_error<BenchmarkFailure>(
115                      "unsupported opcode: unsupported memory access")
116                : Error::success();
117   // These access memory and are handled.
118   case X86II::MRMDestMem:
119   case X86II::MRMSrcMem:
120   case X86II::MRMSrcMem4VOp3:
121   case X86II::MRMSrcMemOp4:
122   case X86II::MRMXm:
123   case X86II::MRM0m:
124   case X86II::MRM1m:
125   case X86II::MRM2m:
126   case X86II::MRM3m:
127   case X86II::MRM4m:
128   case X86II::MRM5m:
129   case X86II::MRM6m:
130   case X86II::MRM7m:
131     return Error::success();
132   // These access memory and are not handled yet.
133   case X86II::RawFrmImm16:
134   case X86II::RawFrmMemOffs:
135   case X86II::RawFrmSrc:
136   case X86II::RawFrmDst:
137   case X86II::RawFrmDstSrc:
138     return make_error<BenchmarkFailure>(
139         "unsupported opcode: non uniform memory access");
140   }
141 }
142 
143 static llvm::Error IsInvalidOpcode(const Instruction &Instr) {
144   const auto OpcodeName = Instr.Name;
145   if ((Instr.Description->TSFlags & X86II::FormMask) == X86II::Pseudo)
146     return llvm::make_error<BenchmarkFailure>(
147         "unsupported opcode: pseudo instruction");
148   if (OpcodeName.startswith("POPF") || OpcodeName.startswith("PUSHF") ||
149       OpcodeName.startswith("ADJCALLSTACK"))
150     return llvm::make_error<BenchmarkFailure>(
151         "unsupported opcode: Push/Pop/AdjCallStack");
152   if (llvm::Error Error = isInvalidMemoryInstr(Instr))
153     return Error;
154   // We do not handle instructions with OPERAND_PCREL.
155   for (const Operand &Op : Instr.Operands)
156     if (Op.isExplicit() &&
157         Op.getExplicitOperandInfo().OperandType == llvm::MCOI::OPERAND_PCREL)
158       return llvm::make_error<BenchmarkFailure>(
159           "unsupported opcode: PC relative operand");
160   // We do not handle second-form X87 instructions. We only handle first-form
161   // ones (_Fp), see comment in X86InstrFPStack.td.
162   for (const Operand &Op : Instr.Operands)
163     if (Op.isReg() && Op.isExplicit() &&
164         Op.getExplicitOperandInfo().RegClass == llvm::X86::RSTRegClassID)
165       return llvm::make_error<BenchmarkFailure>(
166           "unsupported second-form X87 instruction");
167   return llvm::Error::success();
168 }
169 
170 static unsigned getX86FPFlags(const Instruction &Instr) {
171   return Instr.Description->TSFlags & llvm::X86II::FPTypeMask;
172 }
173 
174 namespace {
175 class X86LatencySnippetGenerator : public LatencySnippetGenerator {
176 public:
177   using LatencySnippetGenerator::LatencySnippetGenerator;
178 
179   llvm::Expected<std::vector<CodeTemplate>>
180   generateCodeTemplates(const Instruction &Instr) const override;
181 };
182 } // namespace
183 
184 llvm::Expected<std::vector<CodeTemplate>>
185 X86LatencySnippetGenerator::generateCodeTemplates(
186     const Instruction &Instr) const {
187   if (auto E = IsInvalidOpcode(Instr))
188     return std::move(E);
189 
190   switch (getX86FPFlags(Instr)) {
191   case llvm::X86II::NotFP:
192     return LatencySnippetGenerator::generateCodeTemplates(Instr);
193   case llvm::X86II::ZeroArgFP:
194   case llvm::X86II::OneArgFP:
195   case llvm::X86II::SpecialFP:
196   case llvm::X86II::CompareFP:
197   case llvm::X86II::CondMovFP:
198     return llvm::make_error<BenchmarkFailure>("Unsupported x87 Instruction");
199   case llvm::X86II::OneArgFPRW:
200   case llvm::X86II::TwoArgFP:
201     // These are instructions like
202     //   - `ST(0) = fsqrt(ST(0))` (OneArgFPRW)
203     //   - `ST(0) = ST(0) + ST(i)` (TwoArgFP)
204     // They are intrinsically serial and do not modify the state of the stack.
205     return generateSelfAliasingCodeTemplates(Instr);
206   default:
207     llvm_unreachable("Unknown FP Type!");
208   }
209 }
210 
211 namespace {
212 class X86UopsSnippetGenerator : public UopsSnippetGenerator {
213 public:
214   using UopsSnippetGenerator::UopsSnippetGenerator;
215 
216   llvm::Expected<std::vector<CodeTemplate>>
217   generateCodeTemplates(const Instruction &Instr) const override;
218 };
219 } // namespace
220 
221 llvm::Expected<std::vector<CodeTemplate>>
222 X86UopsSnippetGenerator::generateCodeTemplates(
223     const Instruction &Instr) const {
224   if (auto E = IsInvalidOpcode(Instr))
225     return std::move(E);
226 
227   switch (getX86FPFlags(Instr)) {
228   case llvm::X86II::NotFP:
229     return UopsSnippetGenerator::generateCodeTemplates(Instr);
230   case llvm::X86II::ZeroArgFP:
231   case llvm::X86II::OneArgFP:
232   case llvm::X86II::SpecialFP:
233     return llvm::make_error<BenchmarkFailure>("Unsupported x87 Instruction");
234   case llvm::X86II::OneArgFPRW:
235   case llvm::X86II::TwoArgFP:
236     // These are instructions like
237     //   - `ST(0) = fsqrt(ST(0))` (OneArgFPRW)
238     //   - `ST(0) = ST(0) + ST(i)` (TwoArgFP)
239     // They are intrinsically serial and do not modify the state of the stack.
240     // We generate the same code for latency and uops.
241     return generateSelfAliasingCodeTemplates(Instr);
242   case llvm::X86II::CompareFP:
243   case llvm::X86II::CondMovFP:
244     // We can compute uops for any FP instruction that does not grow or shrink
245     // the stack (either do not touch the stack or push as much as they pop).
246     return generateUnconstrainedCodeTemplates(
247         Instr, "instruction does not grow/shrink the FP stack");
248   default:
249     llvm_unreachable("Unknown FP Type!");
250   }
251 }
252 
253 static unsigned getLoadImmediateOpcode(unsigned RegBitWidth) {
254   switch (RegBitWidth) {
255   case 8:
256     return llvm::X86::MOV8ri;
257   case 16:
258     return llvm::X86::MOV16ri;
259   case 32:
260     return llvm::X86::MOV32ri;
261   case 64:
262     return llvm::X86::MOV64ri;
263   }
264   llvm_unreachable("Invalid Value Width");
265 }
266 
267 // Generates instruction to load an immediate value into a register.
268 static llvm::MCInst loadImmediate(unsigned Reg, unsigned RegBitWidth,
269                                   const llvm::APInt &Value) {
270   if (Value.getBitWidth() > RegBitWidth)
271     llvm_unreachable("Value must fit in the Register");
272   return llvm::MCInstBuilder(getLoadImmediateOpcode(RegBitWidth))
273       .addReg(Reg)
274       .addImm(Value.getZExtValue());
275 }
276 
277 // Allocates scratch memory on the stack.
278 static llvm::MCInst allocateStackSpace(unsigned Bytes) {
279   return llvm::MCInstBuilder(llvm::X86::SUB64ri8)
280       .addReg(llvm::X86::RSP)
281       .addReg(llvm::X86::RSP)
282       .addImm(Bytes);
283 }
284 
285 // Fills scratch memory at offset `OffsetBytes` with value `Imm`.
286 static llvm::MCInst fillStackSpace(unsigned MovOpcode, unsigned OffsetBytes,
287                                    uint64_t Imm) {
288   return llvm::MCInstBuilder(MovOpcode)
289       // Address = ESP
290       .addReg(llvm::X86::RSP) // BaseReg
291       .addImm(1)              // ScaleAmt
292       .addReg(0)              // IndexReg
293       .addImm(OffsetBytes)    // Disp
294       .addReg(0)              // Segment
295       // Immediate.
296       .addImm(Imm);
297 }
298 
299 // Loads scratch memory into register `Reg` using opcode `RMOpcode`.
300 static llvm::MCInst loadToReg(unsigned Reg, unsigned RMOpcode) {
301   return llvm::MCInstBuilder(RMOpcode)
302       .addReg(Reg)
303       // Address = ESP
304       .addReg(llvm::X86::RSP) // BaseReg
305       .addImm(1)              // ScaleAmt
306       .addReg(0)              // IndexReg
307       .addImm(0)              // Disp
308       .addReg(0);             // Segment
309 }
310 
311 // Releases scratch memory.
312 static llvm::MCInst releaseStackSpace(unsigned Bytes) {
313   return llvm::MCInstBuilder(llvm::X86::ADD64ri8)
314       .addReg(llvm::X86::RSP)
315       .addReg(llvm::X86::RSP)
316       .addImm(Bytes);
317 }
318 
319 // Reserves some space on the stack, fills it with the content of the provided
320 // constant and provide methods to load the stack value into a register.
321 namespace {
322 struct ConstantInliner {
323   explicit ConstantInliner(const llvm::APInt &Constant) : Constant_(Constant) {}
324 
325   std::vector<llvm::MCInst> loadAndFinalize(unsigned Reg, unsigned RegBitWidth,
326                                             unsigned Opcode);
327 
328   std::vector<llvm::MCInst> loadX87STAndFinalize(unsigned Reg);
329 
330   std::vector<llvm::MCInst> loadX87FPAndFinalize(unsigned Reg);
331 
332   std::vector<llvm::MCInst> popFlagAndFinalize();
333 
334 private:
335   ConstantInliner &add(const llvm::MCInst &Inst) {
336     Instructions.push_back(Inst);
337     return *this;
338   }
339 
340   void initStack(unsigned Bytes);
341 
342   static constexpr const unsigned kF80Bytes = 10; // 80 bits.
343 
344   llvm::APInt Constant_;
345   std::vector<llvm::MCInst> Instructions;
346 };
347 } // namespace
348 
349 std::vector<llvm::MCInst> ConstantInliner::loadAndFinalize(unsigned Reg,
350                                                            unsigned RegBitWidth,
351                                                            unsigned Opcode) {
352   assert((RegBitWidth & 7) == 0 && "RegBitWidth must be a multiple of 8 bits");
353   initStack(RegBitWidth / 8);
354   add(loadToReg(Reg, Opcode));
355   add(releaseStackSpace(RegBitWidth / 8));
356   return std::move(Instructions);
357 }
358 
359 std::vector<llvm::MCInst> ConstantInliner::loadX87STAndFinalize(unsigned Reg) {
360   initStack(kF80Bytes);
361   add(llvm::MCInstBuilder(llvm::X86::LD_F80m)
362           // Address = ESP
363           .addReg(llvm::X86::RSP) // BaseReg
364           .addImm(1)              // ScaleAmt
365           .addReg(0)              // IndexReg
366           .addImm(0)              // Disp
367           .addReg(0));            // Segment
368   if (Reg != llvm::X86::ST0)
369     add(llvm::MCInstBuilder(llvm::X86::ST_Frr).addReg(Reg));
370   add(releaseStackSpace(kF80Bytes));
371   return std::move(Instructions);
372 }
373 
374 std::vector<llvm::MCInst> ConstantInliner::loadX87FPAndFinalize(unsigned Reg) {
375   initStack(kF80Bytes);
376   add(llvm::MCInstBuilder(llvm::X86::LD_Fp80m)
377           .addReg(Reg)
378           // Address = ESP
379           .addReg(llvm::X86::RSP) // BaseReg
380           .addImm(1)              // ScaleAmt
381           .addReg(0)              // IndexReg
382           .addImm(0)              // Disp
383           .addReg(0));            // Segment
384   add(releaseStackSpace(kF80Bytes));
385   return std::move(Instructions);
386 }
387 
388 std::vector<llvm::MCInst> ConstantInliner::popFlagAndFinalize() {
389   initStack(8);
390   add(llvm::MCInstBuilder(llvm::X86::POPF64));
391   return std::move(Instructions);
392 }
393 
394 void ConstantInliner::initStack(unsigned Bytes) {
395   assert(Constant_.getBitWidth() <= Bytes * 8 &&
396          "Value does not have the correct size");
397   const llvm::APInt WideConstant = Constant_.getBitWidth() < Bytes * 8
398                                        ? Constant_.sext(Bytes * 8)
399                                        : Constant_;
400   add(allocateStackSpace(Bytes));
401   size_t ByteOffset = 0;
402   for (; Bytes - ByteOffset >= 4; ByteOffset += 4)
403     add(fillStackSpace(
404         llvm::X86::MOV32mi, ByteOffset,
405         WideConstant.extractBits(32, ByteOffset * 8).getZExtValue()));
406   if (Bytes - ByteOffset >= 2) {
407     add(fillStackSpace(
408         llvm::X86::MOV16mi, ByteOffset,
409         WideConstant.extractBits(16, ByteOffset * 8).getZExtValue()));
410     ByteOffset += 2;
411   }
412   if (Bytes - ByteOffset >= 1)
413     add(fillStackSpace(
414         llvm::X86::MOV8mi, ByteOffset,
415         WideConstant.extractBits(8, ByteOffset * 8).getZExtValue()));
416 }
417 
418 #include "X86GenExegesis.inc"
419 
420 namespace {
421 class ExegesisX86Target : public ExegesisTarget {
422 public:
423   ExegesisX86Target() : ExegesisTarget(X86CpuPfmCounters) {}
424 
425 private:
426   void addTargetSpecificPasses(llvm::PassManagerBase &PM) const override;
427 
428   unsigned getScratchMemoryRegister(const llvm::Triple &TT) const override;
429 
430   unsigned getMaxMemoryAccessSize() const override { return 64; }
431 
432   void fillMemoryOperands(InstructionTemplate &IT, unsigned Reg,
433                           unsigned Offset) const override;
434 
435   std::vector<llvm::MCInst> setRegTo(const llvm::MCSubtargetInfo &STI,
436                                      unsigned Reg,
437                                      const llvm::APInt &Value) const override;
438 
439   std::unique_ptr<SnippetGenerator>
440   createLatencySnippetGenerator(const LLVMState &State) const override {
441     return llvm::make_unique<X86LatencySnippetGenerator>(State);
442   }
443 
444   std::unique_ptr<SnippetGenerator>
445   createUopsSnippetGenerator(const LLVMState &State) const override {
446     return llvm::make_unique<X86UopsSnippetGenerator>(State);
447   }
448 
449   bool matchesArch(llvm::Triple::ArchType Arch) const override {
450     return Arch == llvm::Triple::x86_64 || Arch == llvm::Triple::x86;
451   }
452 };
453 } // namespace
454 
455 void ExegesisX86Target::addTargetSpecificPasses(
456     llvm::PassManagerBase &PM) const {
457   // Lowers FP pseudo-instructions, e.g. ABS_Fp32 -> ABS_F.
458   PM.add(llvm::createX86FloatingPointStackifierPass());
459 }
460 
461 unsigned
462 ExegesisX86Target::getScratchMemoryRegister(const llvm::Triple &TT) const {
463   if (!TT.isArch64Bit()) {
464     // FIXME: This would require popping from the stack, so we would have to
465     // add some additional setup code.
466     return 0;
467   }
468   return TT.isOSWindows() ? llvm::X86::RCX : llvm::X86::RDI;
469 }
470 
471 void ExegesisX86Target::fillMemoryOperands(InstructionTemplate &IT,
472                                            unsigned Reg,
473                                            unsigned Offset) const {
474   assert(!isInvalidMemoryInstr(IT.Instr) &&
475          "fillMemoryOperands requires a valid memory instruction");
476   int MemOpIdx = X86II::getMemoryOperandNo(IT.Instr.Description->TSFlags);
477   assert(MemOpIdx >= 0 && "invalid memory operand index");
478   // getMemoryOperandNo() ignores tied operands, so we have to add them back.
479   for (unsigned I = 0; I <= static_cast<unsigned>(MemOpIdx); ++I) {
480     const auto &Op = IT.Instr.Operands[I];
481     if (Op.isTied() && Op.getTiedToIndex() < I) {
482       ++MemOpIdx;
483     }
484   }
485   // Now fill in the memory operands.
486   const auto SetOp = [&IT](int OpIdx, const MCOperand &OpVal) {
487     const auto Op = IT.Instr.Operands[OpIdx];
488     assert(Op.isMemory() && Op.isExplicit() && "invalid memory pattern");
489     IT.getValueFor(Op) = OpVal;
490   };
491   SetOp(MemOpIdx + 0, MCOperand::createReg(Reg));    // BaseReg
492   SetOp(MemOpIdx + 1, MCOperand::createImm(1));      // ScaleAmt
493   SetOp(MemOpIdx + 2, MCOperand::createReg(0));      // IndexReg
494   SetOp(MemOpIdx + 3, MCOperand::createImm(Offset)); // Disp
495   SetOp(MemOpIdx + 4, MCOperand::createReg(0));      // Segment
496 }
497 
498 std::vector<llvm::MCInst>
499 ExegesisX86Target::setRegTo(const llvm::MCSubtargetInfo &STI, unsigned Reg,
500                             const llvm::APInt &Value) const {
501   if (llvm::X86::GR8RegClass.contains(Reg))
502     return {loadImmediate(Reg, 8, Value)};
503   if (llvm::X86::GR16RegClass.contains(Reg))
504     return {loadImmediate(Reg, 16, Value)};
505   if (llvm::X86::GR32RegClass.contains(Reg))
506     return {loadImmediate(Reg, 32, Value)};
507   if (llvm::X86::GR64RegClass.contains(Reg))
508     return {loadImmediate(Reg, 64, Value)};
509   ConstantInliner CI(Value);
510   if (llvm::X86::VR64RegClass.contains(Reg))
511     return CI.loadAndFinalize(Reg, 64, llvm::X86::MMX_MOVQ64rm);
512   if (llvm::X86::VR128XRegClass.contains(Reg)) {
513     if (STI.getFeatureBits()[llvm::X86::FeatureAVX512])
514       return CI.loadAndFinalize(Reg, 128, llvm::X86::VMOVDQU32Z128rm);
515     if (STI.getFeatureBits()[llvm::X86::FeatureAVX])
516       return CI.loadAndFinalize(Reg, 128, llvm::X86::VMOVDQUrm);
517     return CI.loadAndFinalize(Reg, 128, llvm::X86::MOVDQUrm);
518   }
519   if (llvm::X86::VR256XRegClass.contains(Reg)) {
520     if (STI.getFeatureBits()[llvm::X86::FeatureAVX512])
521       return CI.loadAndFinalize(Reg, 256, llvm::X86::VMOVDQU32Z256rm);
522     if (STI.getFeatureBits()[llvm::X86::FeatureAVX])
523       return CI.loadAndFinalize(Reg, 256, llvm::X86::VMOVDQUYrm);
524   }
525   if (llvm::X86::VR512RegClass.contains(Reg))
526     if (STI.getFeatureBits()[llvm::X86::FeatureAVX512])
527       return CI.loadAndFinalize(Reg, 512, llvm::X86::VMOVDQU32Zrm);
528   if (llvm::X86::RSTRegClass.contains(Reg)) {
529     return CI.loadX87STAndFinalize(Reg);
530   }
531   if (llvm::X86::RFP32RegClass.contains(Reg) ||
532       llvm::X86::RFP64RegClass.contains(Reg) ||
533       llvm::X86::RFP80RegClass.contains(Reg)) {
534     return CI.loadX87FPAndFinalize(Reg);
535   }
536   if (Reg == llvm::X86::EFLAGS)
537     return CI.popFlagAndFinalize();
538   return {}; // Not yet implemented.
539 }
540 
541 static ExegesisTarget *getTheExegesisX86Target() {
542   static ExegesisX86Target Target;
543   return &Target;
544 }
545 
546 void InitializeX86ExegesisTarget() {
547   ExegesisTarget::registerTarget(getTheExegesisX86Target());
548 }
549 
550 } // namespace exegesis
551 } // namespace llvm
552