1 //===-- Target.cpp ----------------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 #include "../Target.h"
9 
10 #include "../Error.h"
11 #include "../ParallelSnippetGenerator.h"
12 #include "../SerialSnippetGenerator.h"
13 #include "../SnippetGenerator.h"
14 #include "MCTargetDesc/X86BaseInfo.h"
15 #include "MCTargetDesc/X86MCTargetDesc.h"
16 #include "X86.h"
17 #include "X86Counter.h"
18 #include "X86RegisterInfo.h"
19 #include "X86Subtarget.h"
20 #include "llvm/ADT/Sequence.h"
21 #include "llvm/MC/MCInstBuilder.h"
22 #include "llvm/Support/Errc.h"
23 #include "llvm/Support/Error.h"
24 #include "llvm/Support/FormatVariadic.h"
25 
26 #include <memory>
27 #include <string>
28 #include <vector>
29 #if defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64))
30 #include <immintrin.h>
31 #include <intrin.h>
32 #endif
33 
34 namespace llvm {
35 namespace exegesis {
36 
37 static cl::OptionCategory
38     BenchmarkOptions("llvm-exegesis benchmark x86-options");
39 
40 // If a positive value is specified, we are going to use the LBR in
41 // latency-mode.
42 //
43 // Note:
44 //  -  A small value is preferred, but too low a value could result in
45 //     throttling.
46 //  -  A prime number is preferred to avoid always skipping certain blocks.
47 //
48 static cl::opt<unsigned> LbrSamplingPeriod(
49     "x86-lbr-sample-period",
50     cl::desc("The sample period (nbranches/sample), used for LBR sampling"),
51     cl::cat(BenchmarkOptions), cl::init(0));
52 
53 // FIXME: Validates that repetition-mode is loop if LBR is requested.
54 
55 // Returns a non-null reason if we cannot handle the memory references in this
56 // instruction.
57 static const char *isInvalidMemoryInstr(const Instruction &Instr) {
58   switch (Instr.Description.TSFlags & X86II::FormMask) {
59   default:
60     return "Unknown FormMask value";
61   // These have no memory access.
62   case X86II::Pseudo:
63   case X86II::RawFrm:
64   case X86II::AddCCFrm:
65   case X86II::PrefixByte:
66   case X86II::MRMDestReg:
67   case X86II::MRMSrcReg:
68   case X86II::MRMSrcReg4VOp3:
69   case X86II::MRMSrcRegOp4:
70   case X86II::MRMSrcRegCC:
71   case X86II::MRMXrCC:
72   case X86II::MRMr0:
73   case X86II::MRMXr:
74   case X86II::MRM0r:
75   case X86II::MRM1r:
76   case X86II::MRM2r:
77   case X86II::MRM3r:
78   case X86II::MRM4r:
79   case X86II::MRM5r:
80   case X86II::MRM6r:
81   case X86II::MRM7r:
82   case X86II::MRM0X:
83   case X86II::MRM1X:
84   case X86II::MRM2X:
85   case X86II::MRM3X:
86   case X86II::MRM4X:
87   case X86II::MRM5X:
88   case X86II::MRM6X:
89   case X86II::MRM7X:
90   case X86II::MRM_C0:
91   case X86II::MRM_C1:
92   case X86II::MRM_C2:
93   case X86II::MRM_C3:
94   case X86II::MRM_C4:
95   case X86II::MRM_C5:
96   case X86II::MRM_C6:
97   case X86II::MRM_C7:
98   case X86II::MRM_C8:
99   case X86II::MRM_C9:
100   case X86II::MRM_CA:
101   case X86II::MRM_CB:
102   case X86II::MRM_CC:
103   case X86II::MRM_CD:
104   case X86II::MRM_CE:
105   case X86II::MRM_CF:
106   case X86II::MRM_D0:
107   case X86II::MRM_D1:
108   case X86II::MRM_D2:
109   case X86II::MRM_D3:
110   case X86II::MRM_D4:
111   case X86II::MRM_D5:
112   case X86II::MRM_D6:
113   case X86II::MRM_D7:
114   case X86II::MRM_D8:
115   case X86II::MRM_D9:
116   case X86II::MRM_DA:
117   case X86II::MRM_DB:
118   case X86II::MRM_DC:
119   case X86II::MRM_DD:
120   case X86II::MRM_DE:
121   case X86II::MRM_DF:
122   case X86II::MRM_E0:
123   case X86II::MRM_E1:
124   case X86II::MRM_E2:
125   case X86II::MRM_E3:
126   case X86II::MRM_E4:
127   case X86II::MRM_E5:
128   case X86II::MRM_E6:
129   case X86II::MRM_E7:
130   case X86II::MRM_E8:
131   case X86II::MRM_E9:
132   case X86II::MRM_EA:
133   case X86II::MRM_EB:
134   case X86II::MRM_EC:
135   case X86II::MRM_ED:
136   case X86II::MRM_EE:
137   case X86II::MRM_EF:
138   case X86II::MRM_F0:
139   case X86II::MRM_F1:
140   case X86II::MRM_F2:
141   case X86II::MRM_F3:
142   case X86II::MRM_F4:
143   case X86II::MRM_F5:
144   case X86II::MRM_F6:
145   case X86II::MRM_F7:
146   case X86II::MRM_F8:
147   case X86II::MRM_F9:
148   case X86II::MRM_FA:
149   case X86II::MRM_FB:
150   case X86II::MRM_FC:
151   case X86II::MRM_FD:
152   case X86II::MRM_FE:
153   case X86II::MRM_FF:
154   case X86II::RawFrmImm8:
155     return nullptr;
156   case X86II::AddRegFrm:
157     return (Instr.Description.Opcode == X86::POP16r ||
158             Instr.Description.Opcode == X86::POP32r ||
159             Instr.Description.Opcode == X86::PUSH16r ||
160             Instr.Description.Opcode == X86::PUSH32r)
161                ? "unsupported opcode: unsupported memory access"
162                : nullptr;
163   // These access memory and are handled.
164   case X86II::MRMDestMem:
165   case X86II::MRMSrcMem:
166   case X86II::MRMSrcMem4VOp3:
167   case X86II::MRMSrcMemOp4:
168   case X86II::MRMSrcMemCC:
169   case X86II::MRMXmCC:
170   case X86II::MRMXm:
171   case X86II::MRM0m:
172   case X86II::MRM1m:
173   case X86II::MRM2m:
174   case X86II::MRM3m:
175   case X86II::MRM4m:
176   case X86II::MRM5m:
177   case X86II::MRM6m:
178   case X86II::MRM7m:
179     return nullptr;
180   // These access memory and are not handled yet.
181   case X86II::RawFrmImm16:
182   case X86II::RawFrmMemOffs:
183   case X86II::RawFrmSrc:
184   case X86II::RawFrmDst:
185   case X86II::RawFrmDstSrc:
186     return "unsupported opcode: non uniform memory access";
187   }
188 }
189 
190 // If the opcode is invalid, returns a pointer to a character literal indicating
191 // the reason. nullptr indicates a valid opcode.
192 static const char *isInvalidOpcode(const Instruction &Instr) {
193   const auto OpcodeName = Instr.Name;
194   if ((Instr.Description.TSFlags & X86II::FormMask) == X86II::Pseudo)
195     return "unsupported opcode: pseudo instruction";
196   if (OpcodeName.startswith("POP") || OpcodeName.startswith("PUSH") ||
197       OpcodeName.startswith("ADJCALLSTACK") || OpcodeName.startswith("LEAVE"))
198     return "unsupported opcode: Push/Pop/AdjCallStack/Leave";
199   if (const auto reason = isInvalidMemoryInstr(Instr))
200     return reason;
201   // We do not handle instructions with OPERAND_PCREL.
202   for (const Operand &Op : Instr.Operands)
203     if (Op.isExplicit() &&
204         Op.getExplicitOperandInfo().OperandType == MCOI::OPERAND_PCREL)
205       return "unsupported opcode: PC relative operand";
206   // We do not handle second-form X87 instructions. We only handle first-form
207   // ones (_Fp), see comment in X86InstrFPStack.td.
208   for (const Operand &Op : Instr.Operands)
209     if (Op.isReg() && Op.isExplicit() &&
210         Op.getExplicitOperandInfo().RegClass == X86::RSTRegClassID)
211       return "unsupported second-form X87 instruction";
212   return nullptr;
213 }
214 
215 static unsigned getX86FPFlags(const Instruction &Instr) {
216   return Instr.Description.TSFlags & X86II::FPTypeMask;
217 }
218 
219 // Helper to fill a memory operand with a value.
220 static void setMemOp(InstructionTemplate &IT, int OpIdx,
221                      const MCOperand &OpVal) {
222   const auto Op = IT.getInstr().Operands[OpIdx];
223   assert(Op.isExplicit() && "invalid memory pattern");
224   IT.getValueFor(Op) = OpVal;
225 }
226 
227 // Common (latency, uops) code for LEA templates. `GetDestReg` takes the
228 // addressing base and index registers and returns the LEA destination register.
229 static Expected<std::vector<CodeTemplate>> generateLEATemplatesCommon(
230     const Instruction &Instr, const BitVector &ForbiddenRegisters,
231     const LLVMState &State, const SnippetGenerator::Options &Opts,
232     std::function<void(unsigned, unsigned, BitVector &CandidateDestRegs)>
233         RestrictDestRegs) {
234   assert(Instr.Operands.size() == 6 && "invalid LEA");
235   assert(X86II::getMemoryOperandNo(Instr.Description.TSFlags) == 1 &&
236          "invalid LEA");
237 
238   constexpr const int kDestOp = 0;
239   constexpr const int kBaseOp = 1;
240   constexpr const int kIndexOp = 3;
241   auto PossibleDestRegs =
242       Instr.Operands[kDestOp].getRegisterAliasing().sourceBits();
243   remove(PossibleDestRegs, ForbiddenRegisters);
244   auto PossibleBaseRegs =
245       Instr.Operands[kBaseOp].getRegisterAliasing().sourceBits();
246   remove(PossibleBaseRegs, ForbiddenRegisters);
247   auto PossibleIndexRegs =
248       Instr.Operands[kIndexOp].getRegisterAliasing().sourceBits();
249   remove(PossibleIndexRegs, ForbiddenRegisters);
250 
251   const auto &RegInfo = State.getRegInfo();
252   std::vector<CodeTemplate> Result;
253   for (const unsigned BaseReg : PossibleBaseRegs.set_bits()) {
254     for (const unsigned IndexReg : PossibleIndexRegs.set_bits()) {
255       for (int LogScale = 0; LogScale <= 3; ++LogScale) {
256         // FIXME: Add an option for controlling how we explore immediates.
257         for (const int Disp : {0, 42}) {
258           InstructionTemplate IT(&Instr);
259           const int64_t Scale = 1ull << LogScale;
260           setMemOp(IT, 1, MCOperand::createReg(BaseReg));
261           setMemOp(IT, 2, MCOperand::createImm(Scale));
262           setMemOp(IT, 3, MCOperand::createReg(IndexReg));
263           setMemOp(IT, 4, MCOperand::createImm(Disp));
264           // SegmentReg must be 0 for LEA.
265           setMemOp(IT, 5, MCOperand::createReg(0));
266 
267           // Output reg candidates are selected by the caller.
268           auto PossibleDestRegsNow = PossibleDestRegs;
269           RestrictDestRegs(BaseReg, IndexReg, PossibleDestRegsNow);
270           assert(PossibleDestRegsNow.set_bits().begin() !=
271                      PossibleDestRegsNow.set_bits().end() &&
272                  "no remaining registers");
273           setMemOp(
274               IT, 0,
275               MCOperand::createReg(*PossibleDestRegsNow.set_bits().begin()));
276 
277           CodeTemplate CT;
278           CT.Instructions.push_back(std::move(IT));
279           CT.Config = formatv("{3}(%{0}, %{1}, {2})", RegInfo.getName(BaseReg),
280                               RegInfo.getName(IndexReg), Scale, Disp)
281                           .str();
282           Result.push_back(std::move(CT));
283           if (Result.size() >= Opts.MaxConfigsPerOpcode)
284             return std::move(Result);
285         }
286       }
287     }
288   }
289 
290   return std::move(Result);
291 }
292 
293 namespace {
294 class X86SerialSnippetGenerator : public SerialSnippetGenerator {
295 public:
296   using SerialSnippetGenerator::SerialSnippetGenerator;
297 
298   Expected<std::vector<CodeTemplate>>
299   generateCodeTemplates(InstructionTemplate Variant,
300                         const BitVector &ForbiddenRegisters) const override;
301 };
302 } // namespace
303 
304 Expected<std::vector<CodeTemplate>>
305 X86SerialSnippetGenerator::generateCodeTemplates(
306     InstructionTemplate Variant, const BitVector &ForbiddenRegisters) const {
307   const Instruction &Instr = Variant.getInstr();
308 
309   if (const auto reason = isInvalidOpcode(Instr))
310     return make_error<Failure>(reason);
311 
312   // LEA gets special attention.
313   const auto Opcode = Instr.Description.getOpcode();
314   if (Opcode == X86::LEA64r || Opcode == X86::LEA64_32r) {
315     return generateLEATemplatesCommon(
316         Instr, ForbiddenRegisters, State, Opts,
317         [this](unsigned BaseReg, unsigned IndexReg,
318                BitVector &CandidateDestRegs) {
319           // We just select a destination register that aliases the base
320           // register.
321           CandidateDestRegs &=
322               State.getRATC().getRegister(BaseReg).aliasedBits();
323         });
324   }
325 
326   if (Instr.hasMemoryOperands())
327     return make_error<Failure>(
328         "unsupported memory operand in latency measurements");
329 
330   switch (getX86FPFlags(Instr)) {
331   case X86II::NotFP:
332     return SerialSnippetGenerator::generateCodeTemplates(Variant,
333                                                          ForbiddenRegisters);
334   case X86II::ZeroArgFP:
335   case X86II::OneArgFP:
336   case X86II::SpecialFP:
337   case X86II::CompareFP:
338   case X86II::CondMovFP:
339     return make_error<Failure>("Unsupported x87 Instruction");
340   case X86II::OneArgFPRW:
341   case X86II::TwoArgFP:
342     // These are instructions like
343     //   - `ST(0) = fsqrt(ST(0))` (OneArgFPRW)
344     //   - `ST(0) = ST(0) + ST(i)` (TwoArgFP)
345     // They are intrinsically serial and do not modify the state of the stack.
346     return generateSelfAliasingCodeTemplates(Variant);
347   default:
348     llvm_unreachable("Unknown FP Type!");
349   }
350 }
351 
352 namespace {
353 class X86ParallelSnippetGenerator : public ParallelSnippetGenerator {
354 public:
355   using ParallelSnippetGenerator::ParallelSnippetGenerator;
356 
357   Expected<std::vector<CodeTemplate>>
358   generateCodeTemplates(InstructionTemplate Variant,
359                         const BitVector &ForbiddenRegisters) const override;
360 };
361 
362 } // namespace
363 
364 Expected<std::vector<CodeTemplate>>
365 X86ParallelSnippetGenerator::generateCodeTemplates(
366     InstructionTemplate Variant, const BitVector &ForbiddenRegisters) const {
367   const Instruction &Instr = Variant.getInstr();
368 
369   if (const auto reason = isInvalidOpcode(Instr))
370     return make_error<Failure>(reason);
371 
372   // LEA gets special attention.
373   const auto Opcode = Instr.Description.getOpcode();
374   if (Opcode == X86::LEA64r || Opcode == X86::LEA64_32r) {
375     return generateLEATemplatesCommon(
376         Instr, ForbiddenRegisters, State, Opts,
377         [this](unsigned BaseReg, unsigned IndexReg,
378                BitVector &CandidateDestRegs) {
379           // Any destination register that is not used for addressing is fine.
380           remove(CandidateDestRegs,
381                  State.getRATC().getRegister(BaseReg).aliasedBits());
382           remove(CandidateDestRegs,
383                  State.getRATC().getRegister(IndexReg).aliasedBits());
384         });
385   }
386 
387   switch (getX86FPFlags(Instr)) {
388   case X86II::NotFP:
389     return ParallelSnippetGenerator::generateCodeTemplates(Variant,
390                                                            ForbiddenRegisters);
391   case X86II::ZeroArgFP:
392   case X86II::OneArgFP:
393   case X86II::SpecialFP:
394     return make_error<Failure>("Unsupported x87 Instruction");
395   case X86II::OneArgFPRW:
396   case X86II::TwoArgFP:
397     // These are instructions like
398     //   - `ST(0) = fsqrt(ST(0))` (OneArgFPRW)
399     //   - `ST(0) = ST(0) + ST(i)` (TwoArgFP)
400     // They are intrinsically serial and do not modify the state of the stack.
401     // We generate the same code for latency and uops.
402     return generateSelfAliasingCodeTemplates(Variant);
403   case X86II::CompareFP:
404   case X86II::CondMovFP:
405     // We can compute uops for any FP instruction that does not grow or shrink
406     // the stack (either do not touch the stack or push as much as they pop).
407     return generateUnconstrainedCodeTemplates(
408         Variant, "instruction does not grow/shrink the FP stack");
409   default:
410     llvm_unreachable("Unknown FP Type!");
411   }
412 }
413 
414 static unsigned getLoadImmediateOpcode(unsigned RegBitWidth) {
415   switch (RegBitWidth) {
416   case 8:
417     return X86::MOV8ri;
418   case 16:
419     return X86::MOV16ri;
420   case 32:
421     return X86::MOV32ri;
422   case 64:
423     return X86::MOV64ri;
424   }
425   llvm_unreachable("Invalid Value Width");
426 }
427 
428 // Generates instruction to load an immediate value into a register.
429 static MCInst loadImmediate(unsigned Reg, unsigned RegBitWidth,
430                             const APInt &Value) {
431   if (Value.getBitWidth() > RegBitWidth)
432     llvm_unreachable("Value must fit in the Register");
433   return MCInstBuilder(getLoadImmediateOpcode(RegBitWidth))
434       .addReg(Reg)
435       .addImm(Value.getZExtValue());
436 }
437 
438 // Allocates scratch memory on the stack.
439 static MCInst allocateStackSpace(unsigned Bytes) {
440   return MCInstBuilder(X86::SUB64ri8)
441       .addReg(X86::RSP)
442       .addReg(X86::RSP)
443       .addImm(Bytes);
444 }
445 
446 // Fills scratch memory at offset `OffsetBytes` with value `Imm`.
447 static MCInst fillStackSpace(unsigned MovOpcode, unsigned OffsetBytes,
448                              uint64_t Imm) {
449   return MCInstBuilder(MovOpcode)
450       // Address = ESP
451       .addReg(X86::RSP)    // BaseReg
452       .addImm(1)           // ScaleAmt
453       .addReg(0)           // IndexReg
454       .addImm(OffsetBytes) // Disp
455       .addReg(0)           // Segment
456       // Immediate.
457       .addImm(Imm);
458 }
459 
460 // Loads scratch memory into register `Reg` using opcode `RMOpcode`.
461 static MCInst loadToReg(unsigned Reg, unsigned RMOpcode) {
462   return MCInstBuilder(RMOpcode)
463       .addReg(Reg)
464       // Address = ESP
465       .addReg(X86::RSP) // BaseReg
466       .addImm(1)        // ScaleAmt
467       .addReg(0)        // IndexReg
468       .addImm(0)        // Disp
469       .addReg(0);       // Segment
470 }
471 
472 // Releases scratch memory.
473 static MCInst releaseStackSpace(unsigned Bytes) {
474   return MCInstBuilder(X86::ADD64ri8)
475       .addReg(X86::RSP)
476       .addReg(X86::RSP)
477       .addImm(Bytes);
478 }
479 
480 // Reserves some space on the stack, fills it with the content of the provided
481 // constant and provide methods to load the stack value into a register.
482 namespace {
483 struct ConstantInliner {
484   explicit ConstantInliner(const APInt &Constant) : Constant_(Constant) {}
485 
486   std::vector<MCInst> loadAndFinalize(unsigned Reg, unsigned RegBitWidth,
487                                       unsigned Opcode);
488 
489   std::vector<MCInst> loadX87STAndFinalize(unsigned Reg);
490 
491   std::vector<MCInst> loadX87FPAndFinalize(unsigned Reg);
492 
493   std::vector<MCInst> popFlagAndFinalize();
494 
495   std::vector<MCInst> loadImplicitRegAndFinalize(unsigned Opcode,
496                                                  unsigned Value);
497 
498 private:
499   ConstantInliner &add(const MCInst &Inst) {
500     Instructions.push_back(Inst);
501     return *this;
502   }
503 
504   void initStack(unsigned Bytes);
505 
506   static constexpr const unsigned kF80Bytes = 10; // 80 bits.
507 
508   APInt Constant_;
509   std::vector<MCInst> Instructions;
510 };
511 } // namespace
512 
513 std::vector<MCInst> ConstantInliner::loadAndFinalize(unsigned Reg,
514                                                      unsigned RegBitWidth,
515                                                      unsigned Opcode) {
516   assert((RegBitWidth & 7) == 0 && "RegBitWidth must be a multiple of 8 bits");
517   initStack(RegBitWidth / 8);
518   add(loadToReg(Reg, Opcode));
519   add(releaseStackSpace(RegBitWidth / 8));
520   return std::move(Instructions);
521 }
522 
523 std::vector<MCInst> ConstantInliner::loadX87STAndFinalize(unsigned Reg) {
524   initStack(kF80Bytes);
525   add(MCInstBuilder(X86::LD_F80m)
526           // Address = ESP
527           .addReg(X86::RSP) // BaseReg
528           .addImm(1)        // ScaleAmt
529           .addReg(0)        // IndexReg
530           .addImm(0)        // Disp
531           .addReg(0));      // Segment
532   if (Reg != X86::ST0)
533     add(MCInstBuilder(X86::ST_Frr).addReg(Reg));
534   add(releaseStackSpace(kF80Bytes));
535   return std::move(Instructions);
536 }
537 
538 std::vector<MCInst> ConstantInliner::loadX87FPAndFinalize(unsigned Reg) {
539   initStack(kF80Bytes);
540   add(MCInstBuilder(X86::LD_Fp80m)
541           .addReg(Reg)
542           // Address = ESP
543           .addReg(X86::RSP) // BaseReg
544           .addImm(1)        // ScaleAmt
545           .addReg(0)        // IndexReg
546           .addImm(0)        // Disp
547           .addReg(0));      // Segment
548   add(releaseStackSpace(kF80Bytes));
549   return std::move(Instructions);
550 }
551 
552 std::vector<MCInst> ConstantInliner::popFlagAndFinalize() {
553   initStack(8);
554   add(MCInstBuilder(X86::POPF64));
555   return std::move(Instructions);
556 }
557 
558 std::vector<MCInst>
559 ConstantInliner::loadImplicitRegAndFinalize(unsigned Opcode, unsigned Value) {
560   add(allocateStackSpace(4));
561   add(fillStackSpace(X86::MOV32mi, 0, Value)); // Mask all FP exceptions
562   add(MCInstBuilder(Opcode)
563           // Address = ESP
564           .addReg(X86::RSP) // BaseReg
565           .addImm(1)        // ScaleAmt
566           .addReg(0)        // IndexReg
567           .addImm(0)        // Disp
568           .addReg(0));      // Segment
569   add(releaseStackSpace(4));
570   return std::move(Instructions);
571 }
572 
573 void ConstantInliner::initStack(unsigned Bytes) {
574   assert(Constant_.getBitWidth() <= Bytes * 8 &&
575          "Value does not have the correct size");
576   const APInt WideConstant = Constant_.getBitWidth() < Bytes * 8
577                                  ? Constant_.sext(Bytes * 8)
578                                  : Constant_;
579   add(allocateStackSpace(Bytes));
580   size_t ByteOffset = 0;
581   for (; Bytes - ByteOffset >= 4; ByteOffset += 4)
582     add(fillStackSpace(
583         X86::MOV32mi, ByteOffset,
584         WideConstant.extractBits(32, ByteOffset * 8).getZExtValue()));
585   if (Bytes - ByteOffset >= 2) {
586     add(fillStackSpace(
587         X86::MOV16mi, ByteOffset,
588         WideConstant.extractBits(16, ByteOffset * 8).getZExtValue()));
589     ByteOffset += 2;
590   }
591   if (Bytes - ByteOffset >= 1)
592     add(fillStackSpace(
593         X86::MOV8mi, ByteOffset,
594         WideConstant.extractBits(8, ByteOffset * 8).getZExtValue()));
595 }
596 
597 #include "X86GenExegesis.inc"
598 
599 namespace {
600 
601 class X86SavedState : public ExegesisTarget::SavedState {
602 public:
603   X86SavedState() {
604 #ifdef __x86_64__
605 # if defined(_MSC_VER)
606     _fxsave64(FPState);
607     Eflags = __readeflags();
608 # elif defined(__GNUC__)
609     __builtin_ia32_fxsave64(FPState);
610     Eflags = __builtin_ia32_readeflags_u64();
611 # endif
612 #else
613     llvm_unreachable("X86 exegesis running on non-X86 target");
614 #endif
615   }
616 
617   ~X86SavedState() {
618     // Restoring the X87 state does not flush pending exceptions, make sure
619     // these exceptions are flushed now.
620 #ifdef __x86_64__
621 # if defined(_MSC_VER)
622     _clearfp();
623     _fxrstor64(FPState);
624     __writeeflags(Eflags);
625 # elif defined(__GNUC__)
626     asm volatile("fwait");
627     __builtin_ia32_fxrstor64(FPState);
628     __builtin_ia32_writeeflags_u64(Eflags);
629 # endif
630 #else
631     llvm_unreachable("X86 exegesis running on non-X86 target");
632 #endif
633   }
634 
635 private:
636 #ifdef __x86_64__
637   alignas(16) char FPState[512];
638   uint64_t Eflags;
639 #endif
640 };
641 
642 class ExegesisX86Target : public ExegesisTarget {
643 public:
644   ExegesisX86Target() : ExegesisTarget(X86CpuPfmCounters) {}
645 
646   Expected<std::unique_ptr<pfm::Counter>>
647   createCounter(StringRef CounterName, const LLVMState &State) const override {
648     // If LbrSamplingPeriod was provided, then ignore the
649     // CounterName because we only have one for LBR.
650     if (LbrSamplingPeriod > 0) {
651       // Can't use LBR without HAVE_LIBPFM, LIBPFM_HAS_FIELD_CYCLES, or without
652       // __linux__ (for now)
653 #if defined(HAVE_LIBPFM) && defined(LIBPFM_HAS_FIELD_CYCLES) &&                \
654     defined(__linux__)
655       return std::make_unique<X86LbrCounter>(
656           X86LbrPerfEvent(LbrSamplingPeriod));
657 #else
658       return llvm::make_error<llvm::StringError>(
659           "LBR counter requested without HAVE_LIBPFM, LIBPFM_HAS_FIELD_CYCLES, "
660           "or running on Linux.",
661           llvm::errc::invalid_argument);
662 #endif
663     }
664     return ExegesisTarget::createCounter(CounterName, State);
665   }
666 
667 private:
668   void addTargetSpecificPasses(PassManagerBase &PM) const override;
669 
670   unsigned getScratchMemoryRegister(const Triple &TT) const override;
671 
672   unsigned getLoopCounterRegister(const Triple &) const override;
673 
674   unsigned getMaxMemoryAccessSize() const override { return 64; }
675 
676   Error randomizeTargetMCOperand(const Instruction &Instr, const Variable &Var,
677                                  MCOperand &AssignedValue,
678                                  const BitVector &ForbiddenRegs) const override;
679 
680   void fillMemoryOperands(InstructionTemplate &IT, unsigned Reg,
681                           unsigned Offset) const override;
682 
683   void decrementLoopCounterAndJump(MachineBasicBlock &MBB,
684                                    MachineBasicBlock &TargetMBB,
685                                    const MCInstrInfo &MII) const override;
686 
687   std::vector<MCInst> setRegTo(const MCSubtargetInfo &STI, unsigned Reg,
688                                const APInt &Value) const override;
689 
690   ArrayRef<unsigned> getUnavailableRegisters() const override {
691     return makeArrayRef(kUnavailableRegisters,
692                         sizeof(kUnavailableRegisters) /
693                             sizeof(kUnavailableRegisters[0]));
694   }
695 
696   bool allowAsBackToBack(const Instruction &Instr) const override {
697     const unsigned Opcode = Instr.Description.Opcode;
698     return !isInvalidOpcode(Instr) && Opcode != X86::LEA64r &&
699            Opcode != X86::LEA64_32r && Opcode != X86::LEA16r;
700   }
701 
702   std::vector<InstructionTemplate>
703   generateInstructionVariants(const Instruction &Instr,
704                               unsigned MaxConfigsPerOpcode) const override;
705 
706   std::unique_ptr<SnippetGenerator> createSerialSnippetGenerator(
707       const LLVMState &State,
708       const SnippetGenerator::Options &Opts) const override {
709     return std::make_unique<X86SerialSnippetGenerator>(State, Opts);
710   }
711 
712   std::unique_ptr<SnippetGenerator> createParallelSnippetGenerator(
713       const LLVMState &State,
714       const SnippetGenerator::Options &Opts) const override {
715     return std::make_unique<X86ParallelSnippetGenerator>(State, Opts);
716   }
717 
718   bool matchesArch(Triple::ArchType Arch) const override {
719     return Arch == Triple::x86_64 || Arch == Triple::x86;
720   }
721 
722   Error checkFeatureSupport() const override {
723     // LBR is the only feature we conditionally support now.
724     // So if LBR is not requested, then we should be able to run the benchmarks.
725     if (LbrSamplingPeriod == 0)
726       return Error::success();
727 
728 #if defined(__linux__) && defined(HAVE_LIBPFM) &&                              \
729     defined(LIBPFM_HAS_FIELD_CYCLES)
730     // If the kernel supports it, the hardware still may not have it.
731     return X86LbrCounter::checkLbrSupport();
732 #else
733     return llvm::make_error<llvm::StringError>(
734         "LBR not supported on this kernel and/or platform",
735         llvm::errc::not_supported);
736 #endif
737   }
738 
739   std::unique_ptr<SavedState> withSavedState() const override {
740     return std::make_unique<X86SavedState>();
741   }
742 
743   static const unsigned kUnavailableRegisters[4];
744 };
745 
746 // We disable a few registers that cannot be encoded on instructions with a REX
747 // prefix.
748 const unsigned ExegesisX86Target::kUnavailableRegisters[4] = {X86::AH, X86::BH,
749                                                               X86::CH, X86::DH};
750 
751 // We're using one of R8-R15 because these registers are never hardcoded in
752 // instructions (e.g. MOVS writes to EDI, ESI, EDX), so they have less
753 // conflicts.
754 constexpr const unsigned kLoopCounterReg = X86::R8;
755 
756 } // namespace
757 
758 void ExegesisX86Target::addTargetSpecificPasses(PassManagerBase &PM) const {
759   // Lowers FP pseudo-instructions, e.g. ABS_Fp32 -> ABS_F.
760   PM.add(createX86FloatingPointStackifierPass());
761 }
762 
763 unsigned ExegesisX86Target::getScratchMemoryRegister(const Triple &TT) const {
764   if (!TT.isArch64Bit()) {
765     // FIXME: This would require popping from the stack, so we would have to
766     // add some additional setup code.
767     return 0;
768   }
769   return TT.isOSWindows() ? X86::RCX : X86::RDI;
770 }
771 
772 unsigned ExegesisX86Target::getLoopCounterRegister(const Triple &TT) const {
773   if (!TT.isArch64Bit()) {
774     return 0;
775   }
776   return kLoopCounterReg;
777 }
778 
779 Error ExegesisX86Target::randomizeTargetMCOperand(
780     const Instruction &Instr, const Variable &Var, MCOperand &AssignedValue,
781     const BitVector &ForbiddenRegs) const {
782   const Operand &Op = Instr.getPrimaryOperand(Var);
783   switch (Op.getExplicitOperandInfo().OperandType) {
784   case X86::OperandType::OPERAND_ROUNDING_CONTROL:
785     AssignedValue =
786         MCOperand::createImm(randomIndex(X86::STATIC_ROUNDING::TO_ZERO));
787     return Error::success();
788   default:
789     break;
790   }
791   return make_error<Failure>(
792       Twine("unimplemented operand type ")
793           .concat(Twine(Op.getExplicitOperandInfo().OperandType)));
794 }
795 
796 void ExegesisX86Target::fillMemoryOperands(InstructionTemplate &IT,
797                                            unsigned Reg,
798                                            unsigned Offset) const {
799   assert(!isInvalidMemoryInstr(IT.getInstr()) &&
800          "fillMemoryOperands requires a valid memory instruction");
801   int MemOpIdx = X86II::getMemoryOperandNo(IT.getInstr().Description.TSFlags);
802   assert(MemOpIdx >= 0 && "invalid memory operand index");
803   // getMemoryOperandNo() ignores tied operands, so we have to add them back.
804   MemOpIdx += X86II::getOperandBias(IT.getInstr().Description);
805   setMemOp(IT, MemOpIdx + 0, MCOperand::createReg(Reg));    // BaseReg
806   setMemOp(IT, MemOpIdx + 1, MCOperand::createImm(1));      // ScaleAmt
807   setMemOp(IT, MemOpIdx + 2, MCOperand::createReg(0));      // IndexReg
808   setMemOp(IT, MemOpIdx + 3, MCOperand::createImm(Offset)); // Disp
809   setMemOp(IT, MemOpIdx + 4, MCOperand::createReg(0));      // Segment
810 }
811 
812 void ExegesisX86Target::decrementLoopCounterAndJump(
813     MachineBasicBlock &MBB, MachineBasicBlock &TargetMBB,
814     const MCInstrInfo &MII) const {
815   BuildMI(&MBB, DebugLoc(), MII.get(X86::ADD64ri8))
816       .addDef(kLoopCounterReg)
817       .addUse(kLoopCounterReg)
818       .addImm(-1);
819   BuildMI(&MBB, DebugLoc(), MII.get(X86::JCC_1))
820       .addMBB(&TargetMBB)
821       .addImm(X86::COND_NE);
822 }
823 
824 std::vector<MCInst> ExegesisX86Target::setRegTo(const MCSubtargetInfo &STI,
825                                                 unsigned Reg,
826                                                 const APInt &Value) const {
827   if (X86::GR8RegClass.contains(Reg))
828     return {loadImmediate(Reg, 8, Value)};
829   if (X86::GR16RegClass.contains(Reg))
830     return {loadImmediate(Reg, 16, Value)};
831   if (X86::GR32RegClass.contains(Reg))
832     return {loadImmediate(Reg, 32, Value)};
833   if (X86::GR64RegClass.contains(Reg))
834     return {loadImmediate(Reg, 64, Value)};
835   ConstantInliner CI(Value);
836   if (X86::VR64RegClass.contains(Reg))
837     return CI.loadAndFinalize(Reg, 64, X86::MMX_MOVQ64rm);
838   if (X86::VR128XRegClass.contains(Reg)) {
839     if (STI.getFeatureBits()[X86::FeatureAVX512])
840       return CI.loadAndFinalize(Reg, 128, X86::VMOVDQU32Z128rm);
841     if (STI.getFeatureBits()[X86::FeatureAVX])
842       return CI.loadAndFinalize(Reg, 128, X86::VMOVDQUrm);
843     return CI.loadAndFinalize(Reg, 128, X86::MOVDQUrm);
844   }
845   if (X86::VR256XRegClass.contains(Reg)) {
846     if (STI.getFeatureBits()[X86::FeatureAVX512])
847       return CI.loadAndFinalize(Reg, 256, X86::VMOVDQU32Z256rm);
848     if (STI.getFeatureBits()[X86::FeatureAVX])
849       return CI.loadAndFinalize(Reg, 256, X86::VMOVDQUYrm);
850   }
851   if (X86::VR512RegClass.contains(Reg))
852     if (STI.getFeatureBits()[X86::FeatureAVX512])
853       return CI.loadAndFinalize(Reg, 512, X86::VMOVDQU32Zrm);
854   if (X86::RSTRegClass.contains(Reg)) {
855     return CI.loadX87STAndFinalize(Reg);
856   }
857   if (X86::RFP32RegClass.contains(Reg) || X86::RFP64RegClass.contains(Reg) ||
858       X86::RFP80RegClass.contains(Reg)) {
859     return CI.loadX87FPAndFinalize(Reg);
860   }
861   if (Reg == X86::EFLAGS)
862     return CI.popFlagAndFinalize();
863   if (Reg == X86::MXCSR)
864     return CI.loadImplicitRegAndFinalize(
865         STI.getFeatureBits()[X86::FeatureAVX] ? X86::VLDMXCSR : X86::LDMXCSR,
866         0x1f80);
867   if (Reg == X86::FPCW)
868     return CI.loadImplicitRegAndFinalize(X86::FLDCW16m, 0x37f);
869   return {}; // Not yet implemented.
870 }
871 
872 // Instruction can have some variable operands, and we may want to see how
873 // different operands affect performance. So for each operand position,
874 // precompute all the possible choices we might care about,
875 // and greedily generate all the possible combinations of choices.
876 std::vector<InstructionTemplate> ExegesisX86Target::generateInstructionVariants(
877     const Instruction &Instr, unsigned MaxConfigsPerOpcode) const {
878   bool Exploration = false;
879   SmallVector<SmallVector<MCOperand, 1>, 4> VariableChoices;
880   VariableChoices.resize(Instr.Variables.size());
881   for (auto I : llvm::zip(Instr.Variables, VariableChoices)) {
882     const Variable &Var = std::get<0>(I);
883     SmallVectorImpl<MCOperand> &Choices = std::get<1>(I);
884 
885     switch (Instr.getPrimaryOperand(Var).getExplicitOperandInfo().OperandType) {
886     default:
887       // We don't wish to explicitly explore this variable.
888       Choices.emplace_back(); // But add invalid MCOperand to simplify logic.
889       continue;
890     case X86::OperandType::OPERAND_COND_CODE: {
891       Exploration = true;
892       auto CondCodes = seq((int)X86::CondCode::COND_O,
893                            1 + (int)X86::CondCode::LAST_VALID_COND);
894       Choices.reserve(std::distance(CondCodes.begin(), CondCodes.end()));
895       for (int CondCode : CondCodes)
896         Choices.emplace_back(MCOperand::createImm(CondCode));
897       break;
898     }
899     }
900   }
901 
902   // If we don't wish to explore any variables, defer to the baseline method.
903   if (!Exploration)
904     return ExegesisTarget::generateInstructionVariants(Instr,
905                                                        MaxConfigsPerOpcode);
906 
907   std::vector<InstructionTemplate> Variants;
908   size_t NumVariants;
909   CombinationGenerator<MCOperand, decltype(VariableChoices)::value_type, 4> G(
910       VariableChoices);
911 
912   // How many operand combinations can we produce, within the limit?
913   NumVariants = std::min(G.numCombinations(), (size_t)MaxConfigsPerOpcode);
914   // And actually produce all the wanted operand combinations.
915   Variants.reserve(NumVariants);
916   G.generate([&](ArrayRef<MCOperand> State) -> bool {
917     Variants.emplace_back(&Instr);
918     Variants.back().setVariableValues(State);
919     // Did we run out of space for variants?
920     return Variants.size() >= NumVariants;
921   });
922 
923   assert(Variants.size() == NumVariants &&
924          Variants.size() <= MaxConfigsPerOpcode &&
925          "Should not produce too many variants");
926   return Variants;
927 }
928 
929 static ExegesisTarget *getTheExegesisX86Target() {
930   static ExegesisX86Target Target;
931   return &Target;
932 }
933 
934 void InitializeX86ExegesisTarget() {
935   ExegesisTarget::registerTarget(getTheExegesisX86Target());
936 }
937 
938 } // namespace exegesis
939 } // namespace llvm
940