1 //===- X86EvexToVex.cpp ---------------------------------------------------===//
2 // Compress EVEX instructions to VEX encoding when possible to reduce code size
3 //
4 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 // See https://llvm.org/LICENSE.txt for license information.
6 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file
11 /// This file defines the pass that goes over all AVX-512 instructions which
12 /// are encoded using the EVEX prefix and if possible replaces them by their
13 /// corresponding VEX encoding which is usually shorter by 2 bytes.
14 /// EVEX instructions may be encoded via the VEX prefix when the AVX-512
15 /// instruction has a corresponding AVX/AVX2 opcode, when vector length
16 /// accessed by instruction is less than 512 bits and when it does not use
17 //  the xmm or the mask registers or xmm/ymm registers with indexes higher than 15.
18 /// The pass applies code reduction on the generated code for AVX-512 instrs.
19 //
20 //===----------------------------------------------------------------------===//
21 
22 #include "MCTargetDesc/X86BaseInfo.h"
23 #include "MCTargetDesc/X86InstComments.h"
24 #include "X86.h"
25 #include "X86InstrInfo.h"
26 #include "X86Subtarget.h"
27 #include "llvm/ADT/StringRef.h"
28 #include "llvm/CodeGen/MachineFunction.h"
29 #include "llvm/CodeGen/MachineFunctionPass.h"
30 #include "llvm/CodeGen/MachineInstr.h"
31 #include "llvm/CodeGen/MachineOperand.h"
32 #include "llvm/MC/MCInstrDesc.h"
33 #include "llvm/Pass.h"
34 #include <atomic>
35 #include <cassert>
36 #include <cstdint>
37 
38 using namespace llvm;
39 
40 // Including the generated EVEX2VEX tables.
41 struct X86EvexToVexCompressTableEntry {
42   uint16_t EvexOpcode;
43   uint16_t VexOpcode;
44 
45   bool operator<(const X86EvexToVexCompressTableEntry &RHS) const {
46     return EvexOpcode < RHS.EvexOpcode;
47   }
48 
49   friend bool operator<(const X86EvexToVexCompressTableEntry &TE,
50                         unsigned Opc) {
51     return TE.EvexOpcode < Opc;
52   }
53 };
54 #include "X86GenEVEX2VEXTables.inc"
55 
56 #define EVEX2VEX_DESC "Compressing EVEX instrs to VEX encoding when possible"
57 #define EVEX2VEX_NAME "x86-evex-to-vex-compress"
58 
59 #define DEBUG_TYPE EVEX2VEX_NAME
60 
61 namespace {
62 
63 class EvexToVexInstPass : public MachineFunctionPass {
64 
65   /// For EVEX instructions that can be encoded using VEX encoding, replace
66   /// them by the VEX encoding in order to reduce size.
67   bool CompressEvexToVexImpl(MachineInstr &MI) const;
68 
69 public:
70   static char ID;
71 
72   EvexToVexInstPass() : MachineFunctionPass(ID) { }
73 
74   StringRef getPassName() const override { return EVEX2VEX_DESC; }
75 
76   /// Loop over all of the basic blocks, replacing EVEX instructions
77   /// by equivalent VEX instructions when possible for reducing code size.
78   bool runOnMachineFunction(MachineFunction &MF) override;
79 
80   // This pass runs after regalloc and doesn't support VReg operands.
81   MachineFunctionProperties getRequiredProperties() const override {
82     return MachineFunctionProperties().set(
83         MachineFunctionProperties::Property::NoVRegs);
84   }
85 
86 private:
87   /// Machine instruction info used throughout the class.
88   const X86InstrInfo *TII = nullptr;
89 
90   const X86Subtarget *ST = nullptr;
91 };
92 
93 } // end anonymous namespace
94 
95 char EvexToVexInstPass::ID = 0;
96 
97 bool EvexToVexInstPass::runOnMachineFunction(MachineFunction &MF) {
98   TII = MF.getSubtarget<X86Subtarget>().getInstrInfo();
99 
100   ST = &MF.getSubtarget<X86Subtarget>();
101   if (!ST->hasAVX512())
102     return false;
103 
104   bool Changed = false;
105 
106   /// Go over all basic blocks in function and replace
107   /// EVEX encoded instrs by VEX encoding when possible.
108   for (MachineBasicBlock &MBB : MF) {
109 
110     // Traverse the basic block.
111     for (MachineInstr &MI : MBB)
112       Changed |= CompressEvexToVexImpl(MI);
113   }
114 
115   return Changed;
116 }
117 
118 static bool usesExtendedRegister(const MachineInstr &MI) {
119   auto isHiRegIdx = [](unsigned Reg) {
120     // Check for XMM register with indexes between 16 - 31.
121     if (Reg >= X86::XMM16 && Reg <= X86::XMM31)
122       return true;
123 
124     // Check for YMM register with indexes between 16 - 31.
125     if (Reg >= X86::YMM16 && Reg <= X86::YMM31)
126       return true;
127 
128     return false;
129   };
130 
131   // Check that operands are not ZMM regs or
132   // XMM/YMM regs with hi indexes between 16 - 31.
133   for (const MachineOperand &MO : MI.explicit_operands()) {
134     if (!MO.isReg())
135       continue;
136 
137     Register Reg = MO.getReg();
138 
139     assert(!(Reg >= X86::ZMM0 && Reg <= X86::ZMM31) &&
140            "ZMM instructions should not be in the EVEX->VEX tables");
141 
142     if (isHiRegIdx(Reg))
143       return true;
144   }
145 
146   return false;
147 }
148 
149 // Do any custom cleanup needed to finalize the conversion.
150 static bool performCustomAdjustments(MachineInstr &MI, unsigned NewOpc,
151                                      const X86Subtarget *ST) {
152   (void)NewOpc;
153   unsigned Opc = MI.getOpcode();
154   switch (Opc) {
155   case X86::VALIGNDZ128rri:
156   case X86::VALIGNDZ128rmi:
157   case X86::VALIGNQZ128rri:
158   case X86::VALIGNQZ128rmi: {
159     assert((NewOpc == X86::VPALIGNRrri || NewOpc == X86::VPALIGNRrmi) &&
160            "Unexpected new opcode!");
161     unsigned Scale = (Opc == X86::VALIGNQZ128rri ||
162                       Opc == X86::VALIGNQZ128rmi) ? 8 : 4;
163     MachineOperand &Imm = MI.getOperand(MI.getNumExplicitOperands()-1);
164     Imm.setImm(Imm.getImm() * Scale);
165     break;
166   }
167   case X86::VSHUFF32X4Z256rmi:
168   case X86::VSHUFF32X4Z256rri:
169   case X86::VSHUFF64X2Z256rmi:
170   case X86::VSHUFF64X2Z256rri:
171   case X86::VSHUFI32X4Z256rmi:
172   case X86::VSHUFI32X4Z256rri:
173   case X86::VSHUFI64X2Z256rmi:
174   case X86::VSHUFI64X2Z256rri: {
175     assert((NewOpc == X86::VPERM2F128rr || NewOpc == X86::VPERM2I128rr ||
176             NewOpc == X86::VPERM2F128rm || NewOpc == X86::VPERM2I128rm) &&
177            "Unexpected new opcode!");
178     MachineOperand &Imm = MI.getOperand(MI.getNumExplicitOperands()-1);
179     int64_t ImmVal = Imm.getImm();
180     // Set bit 5, move bit 1 to bit 4, copy bit 0.
181     Imm.setImm(0x20 | ((ImmVal & 2) << 3) | (ImmVal & 1));
182     break;
183   }
184   case X86::VRNDSCALEPDZ128rri:
185   case X86::VRNDSCALEPDZ128rmi:
186   case X86::VRNDSCALEPSZ128rri:
187   case X86::VRNDSCALEPSZ128rmi:
188   case X86::VRNDSCALEPDZ256rri:
189   case X86::VRNDSCALEPDZ256rmi:
190   case X86::VRNDSCALEPSZ256rri:
191   case X86::VRNDSCALEPSZ256rmi:
192   case X86::VRNDSCALESDZr:
193   case X86::VRNDSCALESDZm:
194   case X86::VRNDSCALESSZr:
195   case X86::VRNDSCALESSZm:
196   case X86::VRNDSCALESDZr_Int:
197   case X86::VRNDSCALESDZm_Int:
198   case X86::VRNDSCALESSZr_Int:
199   case X86::VRNDSCALESSZm_Int:
200     const MachineOperand &Imm = MI.getOperand(MI.getNumExplicitOperands()-1);
201     int64_t ImmVal = Imm.getImm();
202     // Ensure that only bits 3:0 of the immediate are used.
203     if ((ImmVal & 0xf) != ImmVal)
204       return false;
205     break;
206   }
207 
208   return true;
209 }
210 
211 
212 // For EVEX instructions that can be encoded using VEX encoding
213 // replace them by the VEX encoding in order to reduce size.
214 bool EvexToVexInstPass::CompressEvexToVexImpl(MachineInstr &MI) const {
215   // VEX format.
216   // # of bytes: 0,2,3  1      1      0,1   0,1,2,4  0,1
217   //  [Prefixes] [VEX]  OPCODE ModR/M [SIB] [DISP]  [IMM]
218   //
219   // EVEX format.
220   //  # of bytes: 4    1      1      1      4       / 1         1
221   //  [Prefixes]  EVEX Opcode ModR/M [SIB] [Disp32] / [Disp8*N] [Immediate]
222 
223   const MCInstrDesc &Desc = MI.getDesc();
224 
225   // Check for EVEX instructions only.
226   if ((Desc.TSFlags & X86II::EncodingMask) != X86II::EVEX)
227     return false;
228 
229   // Check for EVEX instructions with mask or broadcast as in these cases
230   // the EVEX prefix is needed in order to carry this information
231   // thus preventing the transformation to VEX encoding.
232   if (Desc.TSFlags & (X86II::EVEX_K | X86II::EVEX_B))
233     return false;
234 
235   // Check for EVEX instructions with L2 set. These instructions are 512-bits
236   // and can't be converted to VEX.
237   if (Desc.TSFlags & X86II::EVEX_L2)
238     return false;
239 
240 #ifndef NDEBUG
241   // Make sure the tables are sorted.
242   static std::atomic<bool> TableChecked(false);
243   if (!TableChecked.load(std::memory_order_relaxed)) {
244     assert(llvm::is_sorted(X86EvexToVex128CompressTable) &&
245            "X86EvexToVex128CompressTable is not sorted!");
246     assert(llvm::is_sorted(X86EvexToVex256CompressTable) &&
247            "X86EvexToVex256CompressTable is not sorted!");
248     TableChecked.store(true, std::memory_order_relaxed);
249   }
250 #endif
251 
252   // Use the VEX.L bit to select the 128 or 256-bit table.
253   ArrayRef<X86EvexToVexCompressTableEntry> Table =
254     (Desc.TSFlags & X86II::VEX_L) ? makeArrayRef(X86EvexToVex256CompressTable)
255                                   : makeArrayRef(X86EvexToVex128CompressTable);
256 
257   const auto *I = llvm::lower_bound(Table, MI.getOpcode());
258   if (I == Table.end() || I->EvexOpcode != MI.getOpcode())
259     return false;
260 
261   unsigned NewOpc = I->VexOpcode;
262 
263   if (usesExtendedRegister(MI))
264     return false;
265 
266   if (!CheckVEXInstPredicate(MI, ST))
267     return false;
268 
269   if (!performCustomAdjustments(MI, NewOpc, ST))
270     return false;
271 
272   MI.setDesc(TII->get(NewOpc));
273   MI.setAsmPrinterFlag(X86::AC_EVEX_2_VEX);
274   return true;
275 }
276 
277 INITIALIZE_PASS(EvexToVexInstPass, EVEX2VEX_NAME, EVEX2VEX_DESC, false, false)
278 
279 FunctionPass *llvm::createX86EvexToVexInsts() {
280   return new EvexToVexInstPass();
281 }
282