1 //===-- Target.cpp ----------------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 #include "../Target.h"
9 
10 #include "../Error.h"
11 #include "../ParallelSnippetGenerator.h"
12 #include "../SerialSnippetGenerator.h"
13 #include "../SnippetGenerator.h"
14 #include "MCTargetDesc/X86BaseInfo.h"
15 #include "MCTargetDesc/X86MCTargetDesc.h"
16 #include "X86.h"
17 #include "X86Counter.h"
18 #include "X86RegisterInfo.h"
19 #include "X86Subtarget.h"
20 #include "llvm/ADT/Sequence.h"
21 #include "llvm/MC/MCInstBuilder.h"
22 #include "llvm/Support/Errc.h"
23 #include "llvm/Support/Error.h"
24 #include "llvm/Support/FormatVariadic.h"
25 #include "llvm/Support/Host.h"
26 
27 #include <memory>
28 #include <string>
29 #include <vector>
30 #if defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64))
31 #include <immintrin.h>
32 #include <intrin.h>
33 #endif
34 
35 namespace llvm {
36 namespace exegesis {
37 
38 static cl::OptionCategory
39     BenchmarkOptions("llvm-exegesis benchmark x86-options");
40 
41 // If a positive value is specified, we are going to use the LBR in
42 // latency-mode.
43 //
44 // Note:
45 //  -  A small value is preferred, but too low a value could result in
46 //     throttling.
47 //  -  A prime number is preferred to avoid always skipping certain blocks.
48 //
49 static cl::opt<unsigned> LbrSamplingPeriod(
50     "x86-lbr-sample-period",
51     cl::desc("The sample period (nbranches/sample), used for LBR sampling"),
52     cl::cat(BenchmarkOptions), cl::init(0));
53 
54 // FIXME: Validates that repetition-mode is loop if LBR is requested.
55 
56 // Returns a non-null reason if we cannot handle the memory references in this
57 // instruction.
58 static const char *isInvalidMemoryInstr(const Instruction &Instr) {
59   switch (Instr.Description.TSFlags & X86II::FormMask) {
60   default:
61     return "Unknown FormMask value";
62   // These have no memory access.
63   case X86II::Pseudo:
64   case X86II::RawFrm:
65   case X86II::AddCCFrm:
66   case X86II::PrefixByte:
67   case X86II::MRMDestReg:
68   case X86II::MRMSrcReg:
69   case X86II::MRMSrcReg4VOp3:
70   case X86II::MRMSrcRegOp4:
71   case X86II::MRMSrcRegCC:
72   case X86II::MRMXrCC:
73   case X86II::MRMr0:
74   case X86II::MRMXr:
75   case X86II::MRM0r:
76   case X86II::MRM1r:
77   case X86II::MRM2r:
78   case X86II::MRM3r:
79   case X86II::MRM4r:
80   case X86II::MRM5r:
81   case X86II::MRM6r:
82   case X86II::MRM7r:
83   case X86II::MRM0X:
84   case X86II::MRM1X:
85   case X86II::MRM2X:
86   case X86II::MRM3X:
87   case X86II::MRM4X:
88   case X86II::MRM5X:
89   case X86II::MRM6X:
90   case X86II::MRM7X:
91   case X86II::MRM_C0:
92   case X86II::MRM_C1:
93   case X86II::MRM_C2:
94   case X86II::MRM_C3:
95   case X86II::MRM_C4:
96   case X86II::MRM_C5:
97   case X86II::MRM_C6:
98   case X86II::MRM_C7:
99   case X86II::MRM_C8:
100   case X86II::MRM_C9:
101   case X86II::MRM_CA:
102   case X86II::MRM_CB:
103   case X86II::MRM_CC:
104   case X86II::MRM_CD:
105   case X86II::MRM_CE:
106   case X86II::MRM_CF:
107   case X86II::MRM_D0:
108   case X86II::MRM_D1:
109   case X86II::MRM_D2:
110   case X86II::MRM_D3:
111   case X86II::MRM_D4:
112   case X86II::MRM_D5:
113   case X86II::MRM_D6:
114   case X86II::MRM_D7:
115   case X86II::MRM_D8:
116   case X86II::MRM_D9:
117   case X86II::MRM_DA:
118   case X86II::MRM_DB:
119   case X86II::MRM_DC:
120   case X86II::MRM_DD:
121   case X86II::MRM_DE:
122   case X86II::MRM_DF:
123   case X86II::MRM_E0:
124   case X86II::MRM_E1:
125   case X86II::MRM_E2:
126   case X86II::MRM_E3:
127   case X86II::MRM_E4:
128   case X86II::MRM_E5:
129   case X86II::MRM_E6:
130   case X86II::MRM_E7:
131   case X86II::MRM_E8:
132   case X86II::MRM_E9:
133   case X86II::MRM_EA:
134   case X86II::MRM_EB:
135   case X86II::MRM_EC:
136   case X86II::MRM_ED:
137   case X86II::MRM_EE:
138   case X86II::MRM_EF:
139   case X86II::MRM_F0:
140   case X86II::MRM_F1:
141   case X86II::MRM_F2:
142   case X86II::MRM_F3:
143   case X86II::MRM_F4:
144   case X86II::MRM_F5:
145   case X86II::MRM_F6:
146   case X86II::MRM_F7:
147   case X86II::MRM_F8:
148   case X86II::MRM_F9:
149   case X86II::MRM_FA:
150   case X86II::MRM_FB:
151   case X86II::MRM_FC:
152   case X86II::MRM_FD:
153   case X86II::MRM_FE:
154   case X86II::MRM_FF:
155   case X86II::RawFrmImm8:
156     return nullptr;
157   case X86II::AddRegFrm:
158     return (Instr.Description.Opcode == X86::POP16r ||
159             Instr.Description.Opcode == X86::POP32r ||
160             Instr.Description.Opcode == X86::PUSH16r ||
161             Instr.Description.Opcode == X86::PUSH32r)
162                ? "unsupported opcode: unsupported memory access"
163                : nullptr;
164   // These access memory and are handled.
165   case X86II::MRMDestMem:
166   case X86II::MRMSrcMem:
167   case X86II::MRMSrcMem4VOp3:
168   case X86II::MRMSrcMemOp4:
169   case X86II::MRMSrcMemCC:
170   case X86II::MRMXmCC:
171   case X86II::MRMXm:
172   case X86II::MRM0m:
173   case X86II::MRM1m:
174   case X86II::MRM2m:
175   case X86II::MRM3m:
176   case X86II::MRM4m:
177   case X86II::MRM5m:
178   case X86II::MRM6m:
179   case X86II::MRM7m:
180     return nullptr;
181   // These access memory and are not handled yet.
182   case X86II::RawFrmImm16:
183   case X86II::RawFrmMemOffs:
184   case X86II::RawFrmSrc:
185   case X86II::RawFrmDst:
186   case X86II::RawFrmDstSrc:
187     return "unsupported opcode: non uniform memory access";
188   }
189 }
190 
191 // If the opcode is invalid, returns a pointer to a character literal indicating
192 // the reason. nullptr indicates a valid opcode.
193 static const char *isInvalidOpcode(const Instruction &Instr) {
194   const auto OpcodeName = Instr.Name;
195   if ((Instr.Description.TSFlags & X86II::FormMask) == X86II::Pseudo)
196     return "unsupported opcode: pseudo instruction";
197   if (OpcodeName.startswith("POP") || OpcodeName.startswith("PUSH") ||
198       OpcodeName.startswith("ADJCALLSTACK") || OpcodeName.startswith("LEAVE"))
199     return "unsupported opcode: Push/Pop/AdjCallStack/Leave";
200   switch (Instr.Description.Opcode) {
201   case X86::LFS16rm:
202   case X86::LFS32rm:
203   case X86::LFS64rm:
204   case X86::LGS16rm:
205   case X86::LGS32rm:
206   case X86::LGS64rm:
207   case X86::LSS16rm:
208   case X86::LSS32rm:
209   case X86::LSS64rm:
210   case X86::SYSENTER:
211     return "unsupported opcode";
212   default:
213     break;
214   }
215   if (const auto reason = isInvalidMemoryInstr(Instr))
216     return reason;
217   // We do not handle instructions with OPERAND_PCREL.
218   for (const Operand &Op : Instr.Operands)
219     if (Op.isExplicit() &&
220         Op.getExplicitOperandInfo().OperandType == MCOI::OPERAND_PCREL)
221       return "unsupported opcode: PC relative operand";
222   // We do not handle second-form X87 instructions. We only handle first-form
223   // ones (_Fp), see comment in X86InstrFPStack.td.
224   for (const Operand &Op : Instr.Operands)
225     if (Op.isReg() && Op.isExplicit() &&
226         Op.getExplicitOperandInfo().RegClass == X86::RSTRegClassID)
227       return "unsupported second-form X87 instruction";
228   return nullptr;
229 }
230 
231 static unsigned getX86FPFlags(const Instruction &Instr) {
232   return Instr.Description.TSFlags & X86II::FPTypeMask;
233 }
234 
235 // Helper to fill a memory operand with a value.
236 static void setMemOp(InstructionTemplate &IT, int OpIdx,
237                      const MCOperand &OpVal) {
238   const auto Op = IT.getInstr().Operands[OpIdx];
239   assert(Op.isExplicit() && "invalid memory pattern");
240   IT.getValueFor(Op) = OpVal;
241 }
242 
243 // Common (latency, uops) code for LEA templates. `GetDestReg` takes the
244 // addressing base and index registers and returns the LEA destination register.
245 static Expected<std::vector<CodeTemplate>> generateLEATemplatesCommon(
246     const Instruction &Instr, const BitVector &ForbiddenRegisters,
247     const LLVMState &State, const SnippetGenerator::Options &Opts,
248     std::function<void(unsigned, unsigned, BitVector &CandidateDestRegs)>
249         RestrictDestRegs) {
250   assert(Instr.Operands.size() == 6 && "invalid LEA");
251   assert(X86II::getMemoryOperandNo(Instr.Description.TSFlags) == 1 &&
252          "invalid LEA");
253 
254   constexpr const int kDestOp = 0;
255   constexpr const int kBaseOp = 1;
256   constexpr const int kIndexOp = 3;
257   auto PossibleDestRegs =
258       Instr.Operands[kDestOp].getRegisterAliasing().sourceBits();
259   remove(PossibleDestRegs, ForbiddenRegisters);
260   auto PossibleBaseRegs =
261       Instr.Operands[kBaseOp].getRegisterAliasing().sourceBits();
262   remove(PossibleBaseRegs, ForbiddenRegisters);
263   auto PossibleIndexRegs =
264       Instr.Operands[kIndexOp].getRegisterAliasing().sourceBits();
265   remove(PossibleIndexRegs, ForbiddenRegisters);
266 
267   const auto &RegInfo = State.getRegInfo();
268   std::vector<CodeTemplate> Result;
269   for (const unsigned BaseReg : PossibleBaseRegs.set_bits()) {
270     for (const unsigned IndexReg : PossibleIndexRegs.set_bits()) {
271       for (int LogScale = 0; LogScale <= 3; ++LogScale) {
272         // FIXME: Add an option for controlling how we explore immediates.
273         for (const int Disp : {0, 42}) {
274           InstructionTemplate IT(&Instr);
275           const int64_t Scale = 1ull << LogScale;
276           setMemOp(IT, 1, MCOperand::createReg(BaseReg));
277           setMemOp(IT, 2, MCOperand::createImm(Scale));
278           setMemOp(IT, 3, MCOperand::createReg(IndexReg));
279           setMemOp(IT, 4, MCOperand::createImm(Disp));
280           // SegmentReg must be 0 for LEA.
281           setMemOp(IT, 5, MCOperand::createReg(0));
282 
283           // Output reg candidates are selected by the caller.
284           auto PossibleDestRegsNow = PossibleDestRegs;
285           RestrictDestRegs(BaseReg, IndexReg, PossibleDestRegsNow);
286           assert(PossibleDestRegsNow.set_bits().begin() !=
287                      PossibleDestRegsNow.set_bits().end() &&
288                  "no remaining registers");
289           setMemOp(
290               IT, 0,
291               MCOperand::createReg(*PossibleDestRegsNow.set_bits().begin()));
292 
293           CodeTemplate CT;
294           CT.Instructions.push_back(std::move(IT));
295           CT.Config = formatv("{3}(%{0}, %{1}, {2})", RegInfo.getName(BaseReg),
296                               RegInfo.getName(IndexReg), Scale, Disp)
297                           .str();
298           Result.push_back(std::move(CT));
299           if (Result.size() >= Opts.MaxConfigsPerOpcode)
300             return std::move(Result);
301         }
302       }
303     }
304   }
305 
306   return std::move(Result);
307 }
308 
309 namespace {
310 class X86SerialSnippetGenerator : public SerialSnippetGenerator {
311 public:
312   using SerialSnippetGenerator::SerialSnippetGenerator;
313 
314   Expected<std::vector<CodeTemplate>>
315   generateCodeTemplates(InstructionTemplate Variant,
316                         const BitVector &ForbiddenRegisters) const override;
317 };
318 } // namespace
319 
320 Expected<std::vector<CodeTemplate>>
321 X86SerialSnippetGenerator::generateCodeTemplates(
322     InstructionTemplate Variant, const BitVector &ForbiddenRegisters) const {
323   const Instruction &Instr = Variant.getInstr();
324 
325   if (const auto reason = isInvalidOpcode(Instr))
326     return make_error<Failure>(reason);
327 
328   // LEA gets special attention.
329   const auto Opcode = Instr.Description.getOpcode();
330   if (Opcode == X86::LEA64r || Opcode == X86::LEA64_32r) {
331     return generateLEATemplatesCommon(
332         Instr, ForbiddenRegisters, State, Opts,
333         [this](unsigned BaseReg, unsigned IndexReg,
334                BitVector &CandidateDestRegs) {
335           // We just select a destination register that aliases the base
336           // register.
337           CandidateDestRegs &=
338               State.getRATC().getRegister(BaseReg).aliasedBits();
339         });
340   }
341 
342   if (Instr.hasMemoryOperands())
343     return make_error<Failure>(
344         "unsupported memory operand in latency measurements");
345 
346   switch (getX86FPFlags(Instr)) {
347   case X86II::NotFP:
348     return SerialSnippetGenerator::generateCodeTemplates(Variant,
349                                                          ForbiddenRegisters);
350   case X86II::ZeroArgFP:
351   case X86II::OneArgFP:
352   case X86II::SpecialFP:
353   case X86II::CompareFP:
354   case X86II::CondMovFP:
355     return make_error<Failure>("Unsupported x87 Instruction");
356   case X86II::OneArgFPRW:
357   case X86II::TwoArgFP:
358     // These are instructions like
359     //   - `ST(0) = fsqrt(ST(0))` (OneArgFPRW)
360     //   - `ST(0) = ST(0) + ST(i)` (TwoArgFP)
361     // They are intrinsically serial and do not modify the state of the stack.
362     return generateSelfAliasingCodeTemplates(Variant);
363   default:
364     llvm_unreachable("Unknown FP Type!");
365   }
366 }
367 
368 namespace {
369 class X86ParallelSnippetGenerator : public ParallelSnippetGenerator {
370 public:
371   using ParallelSnippetGenerator::ParallelSnippetGenerator;
372 
373   Expected<std::vector<CodeTemplate>>
374   generateCodeTemplates(InstructionTemplate Variant,
375                         const BitVector &ForbiddenRegisters) const override;
376 };
377 
378 } // namespace
379 
380 Expected<std::vector<CodeTemplate>>
381 X86ParallelSnippetGenerator::generateCodeTemplates(
382     InstructionTemplate Variant, const BitVector &ForbiddenRegisters) const {
383   const Instruction &Instr = Variant.getInstr();
384 
385   if (const auto reason = isInvalidOpcode(Instr))
386     return make_error<Failure>(reason);
387 
388   // LEA gets special attention.
389   const auto Opcode = Instr.Description.getOpcode();
390   if (Opcode == X86::LEA64r || Opcode == X86::LEA64_32r) {
391     return generateLEATemplatesCommon(
392         Instr, ForbiddenRegisters, State, Opts,
393         [this](unsigned BaseReg, unsigned IndexReg,
394                BitVector &CandidateDestRegs) {
395           // Any destination register that is not used for addressing is fine.
396           remove(CandidateDestRegs,
397                  State.getRATC().getRegister(BaseReg).aliasedBits());
398           remove(CandidateDestRegs,
399                  State.getRATC().getRegister(IndexReg).aliasedBits());
400         });
401   }
402 
403   switch (getX86FPFlags(Instr)) {
404   case X86II::NotFP:
405     return ParallelSnippetGenerator::generateCodeTemplates(Variant,
406                                                            ForbiddenRegisters);
407   case X86II::ZeroArgFP:
408   case X86II::OneArgFP:
409   case X86II::SpecialFP:
410     return make_error<Failure>("Unsupported x87 Instruction");
411   case X86II::OneArgFPRW:
412   case X86II::TwoArgFP:
413     // These are instructions like
414     //   - `ST(0) = fsqrt(ST(0))` (OneArgFPRW)
415     //   - `ST(0) = ST(0) + ST(i)` (TwoArgFP)
416     // They are intrinsically serial and do not modify the state of the stack.
417     // We generate the same code for latency and uops.
418     return generateSelfAliasingCodeTemplates(Variant);
419   case X86II::CompareFP:
420   case X86II::CondMovFP:
421     // We can compute uops for any FP instruction that does not grow or shrink
422     // the stack (either do not touch the stack or push as much as they pop).
423     return generateUnconstrainedCodeTemplates(
424         Variant, "instruction does not grow/shrink the FP stack");
425   default:
426     llvm_unreachable("Unknown FP Type!");
427   }
428 }
429 
430 static unsigned getLoadImmediateOpcode(unsigned RegBitWidth) {
431   switch (RegBitWidth) {
432   case 8:
433     return X86::MOV8ri;
434   case 16:
435     return X86::MOV16ri;
436   case 32:
437     return X86::MOV32ri;
438   case 64:
439     return X86::MOV64ri;
440   }
441   llvm_unreachable("Invalid Value Width");
442 }
443 
444 // Generates instruction to load an immediate value into a register.
445 static MCInst loadImmediate(unsigned Reg, unsigned RegBitWidth,
446                             const APInt &Value) {
447   if (Value.getBitWidth() > RegBitWidth)
448     llvm_unreachable("Value must fit in the Register");
449   return MCInstBuilder(getLoadImmediateOpcode(RegBitWidth))
450       .addReg(Reg)
451       .addImm(Value.getZExtValue());
452 }
453 
454 // Allocates scratch memory on the stack.
455 static MCInst allocateStackSpace(unsigned Bytes) {
456   return MCInstBuilder(X86::SUB64ri8)
457       .addReg(X86::RSP)
458       .addReg(X86::RSP)
459       .addImm(Bytes);
460 }
461 
462 // Fills scratch memory at offset `OffsetBytes` with value `Imm`.
463 static MCInst fillStackSpace(unsigned MovOpcode, unsigned OffsetBytes,
464                              uint64_t Imm) {
465   return MCInstBuilder(MovOpcode)
466       // Address = ESP
467       .addReg(X86::RSP)    // BaseReg
468       .addImm(1)           // ScaleAmt
469       .addReg(0)           // IndexReg
470       .addImm(OffsetBytes) // Disp
471       .addReg(0)           // Segment
472       // Immediate.
473       .addImm(Imm);
474 }
475 
476 // Loads scratch memory into register `Reg` using opcode `RMOpcode`.
477 static MCInst loadToReg(unsigned Reg, unsigned RMOpcode) {
478   return MCInstBuilder(RMOpcode)
479       .addReg(Reg)
480       // Address = ESP
481       .addReg(X86::RSP) // BaseReg
482       .addImm(1)        // ScaleAmt
483       .addReg(0)        // IndexReg
484       .addImm(0)        // Disp
485       .addReg(0);       // Segment
486 }
487 
488 // Releases scratch memory.
489 static MCInst releaseStackSpace(unsigned Bytes) {
490   return MCInstBuilder(X86::ADD64ri8)
491       .addReg(X86::RSP)
492       .addReg(X86::RSP)
493       .addImm(Bytes);
494 }
495 
496 // Reserves some space on the stack, fills it with the content of the provided
497 // constant and provide methods to load the stack value into a register.
498 namespace {
499 struct ConstantInliner {
500   explicit ConstantInliner(const APInt &Constant) : Constant_(Constant) {}
501 
502   std::vector<MCInst> loadAndFinalize(unsigned Reg, unsigned RegBitWidth,
503                                       unsigned Opcode);
504 
505   std::vector<MCInst> loadX87STAndFinalize(unsigned Reg);
506 
507   std::vector<MCInst> loadX87FPAndFinalize(unsigned Reg);
508 
509   std::vector<MCInst> popFlagAndFinalize();
510 
511   std::vector<MCInst> loadImplicitRegAndFinalize(unsigned Opcode,
512                                                  unsigned Value);
513 
514 private:
515   ConstantInliner &add(const MCInst &Inst) {
516     Instructions.push_back(Inst);
517     return *this;
518   }
519 
520   void initStack(unsigned Bytes);
521 
522   static constexpr const unsigned kF80Bytes = 10; // 80 bits.
523 
524   APInt Constant_;
525   std::vector<MCInst> Instructions;
526 };
527 } // namespace
528 
529 std::vector<MCInst> ConstantInliner::loadAndFinalize(unsigned Reg,
530                                                      unsigned RegBitWidth,
531                                                      unsigned Opcode) {
532   assert((RegBitWidth & 7) == 0 && "RegBitWidth must be a multiple of 8 bits");
533   initStack(RegBitWidth / 8);
534   add(loadToReg(Reg, Opcode));
535   add(releaseStackSpace(RegBitWidth / 8));
536   return std::move(Instructions);
537 }
538 
539 std::vector<MCInst> ConstantInliner::loadX87STAndFinalize(unsigned Reg) {
540   initStack(kF80Bytes);
541   add(MCInstBuilder(X86::LD_F80m)
542           // Address = ESP
543           .addReg(X86::RSP) // BaseReg
544           .addImm(1)        // ScaleAmt
545           .addReg(0)        // IndexReg
546           .addImm(0)        // Disp
547           .addReg(0));      // Segment
548   if (Reg != X86::ST0)
549     add(MCInstBuilder(X86::ST_Frr).addReg(Reg));
550   add(releaseStackSpace(kF80Bytes));
551   return std::move(Instructions);
552 }
553 
554 std::vector<MCInst> ConstantInliner::loadX87FPAndFinalize(unsigned Reg) {
555   initStack(kF80Bytes);
556   add(MCInstBuilder(X86::LD_Fp80m)
557           .addReg(Reg)
558           // Address = ESP
559           .addReg(X86::RSP) // BaseReg
560           .addImm(1)        // ScaleAmt
561           .addReg(0)        // IndexReg
562           .addImm(0)        // Disp
563           .addReg(0));      // Segment
564   add(releaseStackSpace(kF80Bytes));
565   return std::move(Instructions);
566 }
567 
568 std::vector<MCInst> ConstantInliner::popFlagAndFinalize() {
569   initStack(8);
570   add(MCInstBuilder(X86::POPF64));
571   return std::move(Instructions);
572 }
573 
574 std::vector<MCInst>
575 ConstantInliner::loadImplicitRegAndFinalize(unsigned Opcode, unsigned Value) {
576   add(allocateStackSpace(4));
577   add(fillStackSpace(X86::MOV32mi, 0, Value)); // Mask all FP exceptions
578   add(MCInstBuilder(Opcode)
579           // Address = ESP
580           .addReg(X86::RSP) // BaseReg
581           .addImm(1)        // ScaleAmt
582           .addReg(0)        // IndexReg
583           .addImm(0)        // Disp
584           .addReg(0));      // Segment
585   add(releaseStackSpace(4));
586   return std::move(Instructions);
587 }
588 
589 void ConstantInliner::initStack(unsigned Bytes) {
590   assert(Constant_.getBitWidth() <= Bytes * 8 &&
591          "Value does not have the correct size");
592   const APInt WideConstant = Constant_.getBitWidth() < Bytes * 8
593                                  ? Constant_.sext(Bytes * 8)
594                                  : Constant_;
595   add(allocateStackSpace(Bytes));
596   size_t ByteOffset = 0;
597   for (; Bytes - ByteOffset >= 4; ByteOffset += 4)
598     add(fillStackSpace(
599         X86::MOV32mi, ByteOffset,
600         WideConstant.extractBits(32, ByteOffset * 8).getZExtValue()));
601   if (Bytes - ByteOffset >= 2) {
602     add(fillStackSpace(
603         X86::MOV16mi, ByteOffset,
604         WideConstant.extractBits(16, ByteOffset * 8).getZExtValue()));
605     ByteOffset += 2;
606   }
607   if (Bytes - ByteOffset >= 1)
608     add(fillStackSpace(
609         X86::MOV8mi, ByteOffset,
610         WideConstant.extractBits(8, ByteOffset * 8).getZExtValue()));
611 }
612 
613 #include "X86GenExegesis.inc"
614 
615 namespace {
616 
617 class X86SavedState : public ExegesisTarget::SavedState {
618 public:
619   X86SavedState() {
620 #ifdef __x86_64__
621 # if defined(_MSC_VER)
622     _fxsave64(FPState);
623     Eflags = __readeflags();
624 # elif defined(__GNUC__)
625     __builtin_ia32_fxsave64(FPState);
626     Eflags = __builtin_ia32_readeflags_u64();
627 # endif
628 #else
629     llvm_unreachable("X86 exegesis running on non-X86 target");
630 #endif
631   }
632 
633   ~X86SavedState() {
634     // Restoring the X87 state does not flush pending exceptions, make sure
635     // these exceptions are flushed now.
636 #ifdef __x86_64__
637 # if defined(_MSC_VER)
638     _clearfp();
639     _fxrstor64(FPState);
640     __writeeflags(Eflags);
641 # elif defined(__GNUC__)
642     asm volatile("fwait");
643     __builtin_ia32_fxrstor64(FPState);
644     __builtin_ia32_writeeflags_u64(Eflags);
645 # endif
646 #else
647     llvm_unreachable("X86 exegesis running on non-X86 target");
648 #endif
649   }
650 
651 private:
652 #ifdef __x86_64__
653   alignas(16) char FPState[512];
654   uint64_t Eflags;
655 #endif
656 };
657 
658 class ExegesisX86Target : public ExegesisTarget {
659 public:
660   ExegesisX86Target() : ExegesisTarget(X86CpuPfmCounters) {}
661 
662   Expected<std::unique_ptr<pfm::Counter>>
663   createCounter(StringRef CounterName, const LLVMState &State) const override {
664     // If LbrSamplingPeriod was provided, then ignore the
665     // CounterName because we only have one for LBR.
666     if (LbrSamplingPeriod > 0) {
667       // Can't use LBR without HAVE_LIBPFM, LIBPFM_HAS_FIELD_CYCLES, or without
668       // __linux__ (for now)
669 #if defined(HAVE_LIBPFM) && defined(LIBPFM_HAS_FIELD_CYCLES) &&                \
670     defined(__linux__)
671       return std::make_unique<X86LbrCounter>(
672           X86LbrPerfEvent(LbrSamplingPeriod));
673 #else
674       return llvm::make_error<llvm::StringError>(
675           "LBR counter requested without HAVE_LIBPFM, LIBPFM_HAS_FIELD_CYCLES, "
676           "or running on Linux.",
677           llvm::errc::invalid_argument);
678 #endif
679     }
680     return ExegesisTarget::createCounter(CounterName, State);
681   }
682 
683 private:
684   void addTargetSpecificPasses(PassManagerBase &PM) const override;
685 
686   unsigned getScratchMemoryRegister(const Triple &TT) const override;
687 
688   unsigned getLoopCounterRegister(const Triple &) const override;
689 
690   unsigned getMaxMemoryAccessSize() const override { return 64; }
691 
692   Error randomizeTargetMCOperand(const Instruction &Instr, const Variable &Var,
693                                  MCOperand &AssignedValue,
694                                  const BitVector &ForbiddenRegs) const override;
695 
696   void fillMemoryOperands(InstructionTemplate &IT, unsigned Reg,
697                           unsigned Offset) const override;
698 
699   void decrementLoopCounterAndJump(MachineBasicBlock &MBB,
700                                    MachineBasicBlock &TargetMBB,
701                                    const MCInstrInfo &MII) const override;
702 
703   std::vector<MCInst> setRegTo(const MCSubtargetInfo &STI, unsigned Reg,
704                                const APInt &Value) const override;
705 
706   ArrayRef<unsigned> getUnavailableRegisters() const override {
707     return makeArrayRef(kUnavailableRegisters,
708                         sizeof(kUnavailableRegisters) /
709                             sizeof(kUnavailableRegisters[0]));
710   }
711 
712   bool allowAsBackToBack(const Instruction &Instr) const override {
713     const unsigned Opcode = Instr.Description.Opcode;
714     return !isInvalidOpcode(Instr) && Opcode != X86::LEA64r &&
715            Opcode != X86::LEA64_32r && Opcode != X86::LEA16r;
716   }
717 
718   std::vector<InstructionTemplate>
719   generateInstructionVariants(const Instruction &Instr,
720                               unsigned MaxConfigsPerOpcode) const override;
721 
722   std::unique_ptr<SnippetGenerator> createSerialSnippetGenerator(
723       const LLVMState &State,
724       const SnippetGenerator::Options &Opts) const override {
725     return std::make_unique<X86SerialSnippetGenerator>(State, Opts);
726   }
727 
728   std::unique_ptr<SnippetGenerator> createParallelSnippetGenerator(
729       const LLVMState &State,
730       const SnippetGenerator::Options &Opts) const override {
731     return std::make_unique<X86ParallelSnippetGenerator>(State, Opts);
732   }
733 
734   bool matchesArch(Triple::ArchType Arch) const override {
735     return Arch == Triple::x86_64 || Arch == Triple::x86;
736   }
737 
738   Error checkFeatureSupport() const override {
739     // LBR is the only feature we conditionally support now.
740     // So if LBR is not requested, then we should be able to run the benchmarks.
741     if (LbrSamplingPeriod == 0)
742       return Error::success();
743 
744 #if defined(__linux__) && defined(HAVE_LIBPFM) &&                              \
745     defined(LIBPFM_HAS_FIELD_CYCLES)
746       // FIXME: Fix this.
747       // https://bugs.llvm.org/show_bug.cgi?id=48918
748       // For now, only do the check if we see an Intel machine because
749       // the counter uses some intel-specific magic and it could
750       // be confuse and think an AMD machine actually has LBR support.
751 #if defined(__i386__) || defined(_M_IX86) || defined(__x86_64__) ||            \
752     defined(_M_X64)
753     using namespace sys::detail::x86;
754 
755     if (getVendorSignature() == VendorSignatures::GENUINE_INTEL)
756       // If the kernel supports it, the hardware still may not have it.
757       return X86LbrCounter::checkLbrSupport();
758 #else
759     llvm_unreachable("Running X86 exegesis on non-X86 target");
760 #endif
761 #endif
762     return llvm::make_error<llvm::StringError>(
763         "LBR not supported on this kernel and/or platform",
764         llvm::errc::not_supported);
765   }
766 
767   std::unique_ptr<SavedState> withSavedState() const override {
768     return std::make_unique<X86SavedState>();
769   }
770 
771   static const unsigned kUnavailableRegisters[4];
772 };
773 
774 // We disable a few registers that cannot be encoded on instructions with a REX
775 // prefix.
776 const unsigned ExegesisX86Target::kUnavailableRegisters[4] = {X86::AH, X86::BH,
777                                                               X86::CH, X86::DH};
778 
779 // We're using one of R8-R15 because these registers are never hardcoded in
780 // instructions (e.g. MOVS writes to EDI, ESI, EDX), so they have less
781 // conflicts.
782 constexpr const unsigned kLoopCounterReg = X86::R8;
783 
784 } // namespace
785 
786 void ExegesisX86Target::addTargetSpecificPasses(PassManagerBase &PM) const {
787   // Lowers FP pseudo-instructions, e.g. ABS_Fp32 -> ABS_F.
788   PM.add(createX86FloatingPointStackifierPass());
789 }
790 
791 unsigned ExegesisX86Target::getScratchMemoryRegister(const Triple &TT) const {
792   if (!TT.isArch64Bit()) {
793     // FIXME: This would require popping from the stack, so we would have to
794     // add some additional setup code.
795     return 0;
796   }
797   return TT.isOSWindows() ? X86::RCX : X86::RDI;
798 }
799 
800 unsigned ExegesisX86Target::getLoopCounterRegister(const Triple &TT) const {
801   if (!TT.isArch64Bit()) {
802     return 0;
803   }
804   return kLoopCounterReg;
805 }
806 
807 Error ExegesisX86Target::randomizeTargetMCOperand(
808     const Instruction &Instr, const Variable &Var, MCOperand &AssignedValue,
809     const BitVector &ForbiddenRegs) const {
810   const Operand &Op = Instr.getPrimaryOperand(Var);
811   switch (Op.getExplicitOperandInfo().OperandType) {
812   case X86::OperandType::OPERAND_ROUNDING_CONTROL:
813     AssignedValue =
814         MCOperand::createImm(randomIndex(X86::STATIC_ROUNDING::TO_ZERO));
815     return Error::success();
816   default:
817     break;
818   }
819   return make_error<Failure>(
820       Twine("unimplemented operand type ")
821           .concat(Twine(Op.getExplicitOperandInfo().OperandType)));
822 }
823 
824 void ExegesisX86Target::fillMemoryOperands(InstructionTemplate &IT,
825                                            unsigned Reg,
826                                            unsigned Offset) const {
827   assert(!isInvalidMemoryInstr(IT.getInstr()) &&
828          "fillMemoryOperands requires a valid memory instruction");
829   int MemOpIdx = X86II::getMemoryOperandNo(IT.getInstr().Description.TSFlags);
830   assert(MemOpIdx >= 0 && "invalid memory operand index");
831   // getMemoryOperandNo() ignores tied operands, so we have to add them back.
832   MemOpIdx += X86II::getOperandBias(IT.getInstr().Description);
833   setMemOp(IT, MemOpIdx + 0, MCOperand::createReg(Reg));    // BaseReg
834   setMemOp(IT, MemOpIdx + 1, MCOperand::createImm(1));      // ScaleAmt
835   setMemOp(IT, MemOpIdx + 2, MCOperand::createReg(0));      // IndexReg
836   setMemOp(IT, MemOpIdx + 3, MCOperand::createImm(Offset)); // Disp
837   setMemOp(IT, MemOpIdx + 4, MCOperand::createReg(0));      // Segment
838 }
839 
840 void ExegesisX86Target::decrementLoopCounterAndJump(
841     MachineBasicBlock &MBB, MachineBasicBlock &TargetMBB,
842     const MCInstrInfo &MII) const {
843   BuildMI(&MBB, DebugLoc(), MII.get(X86::ADD64ri8))
844       .addDef(kLoopCounterReg)
845       .addUse(kLoopCounterReg)
846       .addImm(-1);
847   BuildMI(&MBB, DebugLoc(), MII.get(X86::JCC_1))
848       .addMBB(&TargetMBB)
849       .addImm(X86::COND_NE);
850 }
851 
852 std::vector<MCInst> ExegesisX86Target::setRegTo(const MCSubtargetInfo &STI,
853                                                 unsigned Reg,
854                                                 const APInt &Value) const {
855   if (X86::GR8RegClass.contains(Reg))
856     return {loadImmediate(Reg, 8, Value)};
857   if (X86::GR16RegClass.contains(Reg))
858     return {loadImmediate(Reg, 16, Value)};
859   if (X86::GR32RegClass.contains(Reg))
860     return {loadImmediate(Reg, 32, Value)};
861   if (X86::GR64RegClass.contains(Reg))
862     return {loadImmediate(Reg, 64, Value)};
863   ConstantInliner CI(Value);
864   if (X86::VR64RegClass.contains(Reg))
865     return CI.loadAndFinalize(Reg, 64, X86::MMX_MOVQ64rm);
866   if (X86::VR128XRegClass.contains(Reg)) {
867     if (STI.getFeatureBits()[X86::FeatureAVX512])
868       return CI.loadAndFinalize(Reg, 128, X86::VMOVDQU32Z128rm);
869     if (STI.getFeatureBits()[X86::FeatureAVX])
870       return CI.loadAndFinalize(Reg, 128, X86::VMOVDQUrm);
871     return CI.loadAndFinalize(Reg, 128, X86::MOVDQUrm);
872   }
873   if (X86::VR256XRegClass.contains(Reg)) {
874     if (STI.getFeatureBits()[X86::FeatureAVX512])
875       return CI.loadAndFinalize(Reg, 256, X86::VMOVDQU32Z256rm);
876     if (STI.getFeatureBits()[X86::FeatureAVX])
877       return CI.loadAndFinalize(Reg, 256, X86::VMOVDQUYrm);
878   }
879   if (X86::VR512RegClass.contains(Reg))
880     if (STI.getFeatureBits()[X86::FeatureAVX512])
881       return CI.loadAndFinalize(Reg, 512, X86::VMOVDQU32Zrm);
882   if (X86::RSTRegClass.contains(Reg)) {
883     return CI.loadX87STAndFinalize(Reg);
884   }
885   if (X86::RFP32RegClass.contains(Reg) || X86::RFP64RegClass.contains(Reg) ||
886       X86::RFP80RegClass.contains(Reg)) {
887     return CI.loadX87FPAndFinalize(Reg);
888   }
889   if (Reg == X86::EFLAGS)
890     return CI.popFlagAndFinalize();
891   if (Reg == X86::MXCSR)
892     return CI.loadImplicitRegAndFinalize(
893         STI.getFeatureBits()[X86::FeatureAVX] ? X86::VLDMXCSR : X86::LDMXCSR,
894         0x1f80);
895   if (Reg == X86::FPCW)
896     return CI.loadImplicitRegAndFinalize(X86::FLDCW16m, 0x37f);
897   return {}; // Not yet implemented.
898 }
899 
900 // Instruction can have some variable operands, and we may want to see how
901 // different operands affect performance. So for each operand position,
902 // precompute all the possible choices we might care about,
903 // and greedily generate all the possible combinations of choices.
904 std::vector<InstructionTemplate> ExegesisX86Target::generateInstructionVariants(
905     const Instruction &Instr, unsigned MaxConfigsPerOpcode) const {
906   bool Exploration = false;
907   SmallVector<SmallVector<MCOperand, 1>, 4> VariableChoices;
908   VariableChoices.resize(Instr.Variables.size());
909   for (auto I : llvm::zip(Instr.Variables, VariableChoices)) {
910     const Variable &Var = std::get<0>(I);
911     SmallVectorImpl<MCOperand> &Choices = std::get<1>(I);
912 
913     switch (Instr.getPrimaryOperand(Var).getExplicitOperandInfo().OperandType) {
914     default:
915       // We don't wish to explicitly explore this variable.
916       Choices.emplace_back(); // But add invalid MCOperand to simplify logic.
917       continue;
918     case X86::OperandType::OPERAND_COND_CODE: {
919       Exploration = true;
920       auto CondCodes = seq((int)X86::CondCode::COND_O,
921                            1 + (int)X86::CondCode::LAST_VALID_COND);
922       Choices.reserve(std::distance(CondCodes.begin(), CondCodes.end()));
923       for (int CondCode : CondCodes)
924         Choices.emplace_back(MCOperand::createImm(CondCode));
925       break;
926     }
927     }
928   }
929 
930   // If we don't wish to explore any variables, defer to the baseline method.
931   if (!Exploration)
932     return ExegesisTarget::generateInstructionVariants(Instr,
933                                                        MaxConfigsPerOpcode);
934 
935   std::vector<InstructionTemplate> Variants;
936   size_t NumVariants;
937   CombinationGenerator<MCOperand, decltype(VariableChoices)::value_type, 4> G(
938       VariableChoices);
939 
940   // How many operand combinations can we produce, within the limit?
941   NumVariants = std::min(G.numCombinations(), (size_t)MaxConfigsPerOpcode);
942   // And actually produce all the wanted operand combinations.
943   Variants.reserve(NumVariants);
944   G.generate([&](ArrayRef<MCOperand> State) -> bool {
945     Variants.emplace_back(&Instr);
946     Variants.back().setVariableValues(State);
947     // Did we run out of space for variants?
948     return Variants.size() >= NumVariants;
949   });
950 
951   assert(Variants.size() == NumVariants &&
952          Variants.size() <= MaxConfigsPerOpcode &&
953          "Should not produce too many variants");
954   return Variants;
955 }
956 
957 static ExegesisTarget *getTheExegesisX86Target() {
958   static ExegesisX86Target Target;
959   return &Target;
960 }
961 
962 void InitializeX86ExegesisTarget() {
963   ExegesisTarget::registerTarget(getTheExegesisX86Target());
964 }
965 
966 } // namespace exegesis
967 } // namespace llvm
968