1 //===- AArch64MacroFusion.cpp - AArch64 Macro Fusion ----------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file This file contains the AArch64 implementation of the DAG scheduling
11 ///  mutation to pair instructions back to back.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "AArch64Subtarget.h"
16 #include "llvm/CodeGen/MacroFusion.h"
17 #include "llvm/CodeGen/TargetInstrInfo.h"
18 
19 using namespace llvm;
20 
21 namespace {
22 
23 /// CMN, CMP, TST followed by Bcc
isArithmeticBccPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)24 static bool isArithmeticBccPair(const MachineInstr *FirstMI,
25                                 const MachineInstr &SecondMI) {
26   if (SecondMI.getOpcode() != AArch64::Bcc)
27     return false;
28 
29   // Assume the 1st instr to be a wildcard if it is unspecified.
30   if (FirstMI == nullptr)
31     return true;
32 
33   switch (FirstMI->getOpcode()) {
34   case AArch64::ADDSWri:
35   case AArch64::ADDSWrr:
36   case AArch64::ADDSXri:
37   case AArch64::ADDSXrr:
38   case AArch64::ANDSWri:
39   case AArch64::ANDSWrr:
40   case AArch64::ANDSXri:
41   case AArch64::ANDSXrr:
42   case AArch64::SUBSWri:
43   case AArch64::SUBSWrr:
44   case AArch64::SUBSXri:
45   case AArch64::SUBSXrr:
46   case AArch64::BICSWrr:
47   case AArch64::BICSXrr:
48     return true;
49   case AArch64::ADDSWrs:
50   case AArch64::ADDSXrs:
51   case AArch64::ANDSWrs:
52   case AArch64::ANDSXrs:
53   case AArch64::SUBSWrs:
54   case AArch64::SUBSXrs:
55   case AArch64::BICSWrs:
56   case AArch64::BICSXrs:
57     // Shift value can be 0 making these behave like the "rr" variant...
58     return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
59   }
60 
61   return false;
62 }
63 
64 /// ALU operations followed by CBZ/CBNZ.
isArithmeticCbzPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)65 static bool isArithmeticCbzPair(const MachineInstr *FirstMI,
66                                 const MachineInstr &SecondMI) {
67   if (SecondMI.getOpcode() != AArch64::CBZW &&
68       SecondMI.getOpcode() != AArch64::CBZX &&
69       SecondMI.getOpcode() != AArch64::CBNZW &&
70       SecondMI.getOpcode() != AArch64::CBNZX)
71     return false;
72 
73   // Assume the 1st instr to be a wildcard if it is unspecified.
74   if (FirstMI == nullptr)
75     return true;
76 
77   switch (FirstMI->getOpcode()) {
78   case AArch64::ADDWri:
79   case AArch64::ADDWrr:
80   case AArch64::ADDXri:
81   case AArch64::ADDXrr:
82   case AArch64::ANDWri:
83   case AArch64::ANDWrr:
84   case AArch64::ANDXri:
85   case AArch64::ANDXrr:
86   case AArch64::EORWri:
87   case AArch64::EORWrr:
88   case AArch64::EORXri:
89   case AArch64::EORXrr:
90   case AArch64::ORRWri:
91   case AArch64::ORRWrr:
92   case AArch64::ORRXri:
93   case AArch64::ORRXrr:
94   case AArch64::SUBWri:
95   case AArch64::SUBWrr:
96   case AArch64::SUBXri:
97   case AArch64::SUBXrr:
98     return true;
99   case AArch64::ADDWrs:
100   case AArch64::ADDXrs:
101   case AArch64::ANDWrs:
102   case AArch64::ANDXrs:
103   case AArch64::SUBWrs:
104   case AArch64::SUBXrs:
105   case AArch64::BICWrs:
106   case AArch64::BICXrs:
107     // Shift value can be 0 making these behave like the "rr" variant...
108     return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
109   }
110 
111   return false;
112 }
113 
114 /// AES crypto encoding or decoding.
isAESPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)115 static bool isAESPair(const MachineInstr *FirstMI,
116                       const MachineInstr &SecondMI) {
117   // Assume the 1st instr to be a wildcard if it is unspecified.
118   switch (SecondMI.getOpcode()) {
119   // AES encode.
120   case AArch64::AESMCrr:
121   case AArch64::AESMCrrTied:
122     return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESErr;
123   // AES decode.
124   case AArch64::AESIMCrr:
125   case AArch64::AESIMCrrTied:
126     return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESDrr;
127   }
128 
129   return false;
130 }
131 
132 /// AESE/AESD/PMULL + EOR.
isCryptoEORPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)133 static bool isCryptoEORPair(const MachineInstr *FirstMI,
134                             const MachineInstr &SecondMI) {
135   if (SecondMI.getOpcode() != AArch64::EORv16i8)
136     return false;
137 
138   // Assume the 1st instr to be a wildcard if it is unspecified.
139   if (FirstMI == nullptr)
140     return true;
141 
142   switch (FirstMI->getOpcode()) {
143   case AArch64::AESErr:
144   case AArch64::AESDrr:
145   case AArch64::PMULLv16i8:
146   case AArch64::PMULLv8i8:
147   case AArch64::PMULLv1i64:
148   case AArch64::PMULLv2i64:
149     return true;
150   }
151 
152   return false;
153 }
154 
155 /// Literal generation.
isLiteralsPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)156 static bool isLiteralsPair(const MachineInstr *FirstMI,
157                            const MachineInstr &SecondMI) {
158   // Assume the 1st instr to be a wildcard if it is unspecified.
159 
160   // PC relative address.
161   if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::ADRP) &&
162       SecondMI.getOpcode() == AArch64::ADDXri)
163     return true;
164 
165   // 32 bit immediate.
166   if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZWi) &&
167       (SecondMI.getOpcode() == AArch64::MOVKWi &&
168        SecondMI.getOperand(3).getImm() == 16))
169     return true;
170 
171   // Lower half of 64 bit immediate.
172   if((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZXi) &&
173      (SecondMI.getOpcode() == AArch64::MOVKXi &&
174       SecondMI.getOperand(3).getImm() == 16))
175     return true;
176 
177   // Upper half of 64 bit immediate.
178   if ((FirstMI == nullptr ||
179        (FirstMI->getOpcode() == AArch64::MOVKXi &&
180         FirstMI->getOperand(3).getImm() == 32)) &&
181       (SecondMI.getOpcode() == AArch64::MOVKXi &&
182        SecondMI.getOperand(3).getImm() == 48))
183     return true;
184 
185   return false;
186 }
187 
188 /// Fuse address generation and loads or stores.
isAddressLdStPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)189 static bool isAddressLdStPair(const MachineInstr *FirstMI,
190                               const MachineInstr &SecondMI) {
191   switch (SecondMI.getOpcode()) {
192   case AArch64::STRBBui:
193   case AArch64::STRBui:
194   case AArch64::STRDui:
195   case AArch64::STRHHui:
196   case AArch64::STRHui:
197   case AArch64::STRQui:
198   case AArch64::STRSui:
199   case AArch64::STRWui:
200   case AArch64::STRXui:
201   case AArch64::LDRBBui:
202   case AArch64::LDRBui:
203   case AArch64::LDRDui:
204   case AArch64::LDRHHui:
205   case AArch64::LDRHui:
206   case AArch64::LDRQui:
207   case AArch64::LDRSui:
208   case AArch64::LDRWui:
209   case AArch64::LDRXui:
210   case AArch64::LDRSBWui:
211   case AArch64::LDRSBXui:
212   case AArch64::LDRSHWui:
213   case AArch64::LDRSHXui:
214   case AArch64::LDRSWui:
215     // Assume the 1st instr to be a wildcard if it is unspecified.
216     if (FirstMI == nullptr)
217       return true;
218 
219    switch (FirstMI->getOpcode()) {
220     case AArch64::ADR:
221       return SecondMI.getOperand(2).getImm() == 0;
222     case AArch64::ADRP:
223       return true;
224     }
225   }
226 
227   return false;
228 }
229 
230 /// Compare and conditional select.
isCCSelectPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)231 static bool isCCSelectPair(const MachineInstr *FirstMI,
232                            const MachineInstr &SecondMI) {
233   // 32 bits
234   if (SecondMI.getOpcode() == AArch64::CSELWr) {
235     // Assume the 1st instr to be a wildcard if it is unspecified.
236     if (FirstMI == nullptr)
237       return true;
238 
239     if (FirstMI->definesRegister(AArch64::WZR))
240       switch (FirstMI->getOpcode()) {
241       case AArch64::SUBSWrs:
242         return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
243       case AArch64::SUBSWrx:
244         return !AArch64InstrInfo::hasExtendedReg(*FirstMI);
245       case AArch64::SUBSWrr:
246       case AArch64::SUBSWri:
247         return true;
248       }
249   }
250 
251   // 64 bits
252   if (SecondMI.getOpcode() == AArch64::CSELXr) {
253     // Assume the 1st instr to be a wildcard if it is unspecified.
254     if (FirstMI == nullptr)
255       return true;
256 
257     if (FirstMI->definesRegister(AArch64::XZR))
258       switch (FirstMI->getOpcode()) {
259       case AArch64::SUBSXrs:
260         return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
261       case AArch64::SUBSXrx:
262       case AArch64::SUBSXrx64:
263         return !AArch64InstrInfo::hasExtendedReg(*FirstMI);
264       case AArch64::SUBSXrr:
265       case AArch64::SUBSXri:
266         return true;
267       }
268   }
269 
270   return false;
271 }
272 
273 // Arithmetic and logic.
isArithmeticLogicPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)274 static bool isArithmeticLogicPair(const MachineInstr *FirstMI,
275                                   const MachineInstr &SecondMI) {
276   if (AArch64InstrInfo::hasShiftedReg(SecondMI))
277     return false;
278 
279   switch (SecondMI.getOpcode()) {
280   // Arithmetic
281   case AArch64::ADDWrr:
282   case AArch64::ADDXrr:
283   case AArch64::SUBWrr:
284   case AArch64::SUBXrr:
285   case AArch64::ADDWrs:
286   case AArch64::ADDXrs:
287   case AArch64::SUBWrs:
288   case AArch64::SUBXrs:
289   // Logic
290   case AArch64::ANDWrr:
291   case AArch64::ANDXrr:
292   case AArch64::BICWrr:
293   case AArch64::BICXrr:
294   case AArch64::EONWrr:
295   case AArch64::EONXrr:
296   case AArch64::EORWrr:
297   case AArch64::EORXrr:
298   case AArch64::ORNWrr:
299   case AArch64::ORNXrr:
300   case AArch64::ORRWrr:
301   case AArch64::ORRXrr:
302   case AArch64::ANDWrs:
303   case AArch64::ANDXrs:
304   case AArch64::BICWrs:
305   case AArch64::BICXrs:
306   case AArch64::EONWrs:
307   case AArch64::EONXrs:
308   case AArch64::EORWrs:
309   case AArch64::EORXrs:
310   case AArch64::ORNWrs:
311   case AArch64::ORNXrs:
312   case AArch64::ORRWrs:
313   case AArch64::ORRXrs:
314     // Assume the 1st instr to be a wildcard if it is unspecified.
315     if (FirstMI == nullptr)
316       return true;
317 
318     // Arithmetic
319     switch (FirstMI->getOpcode()) {
320     case AArch64::ADDWrr:
321     case AArch64::ADDXrr:
322     case AArch64::ADDSWrr:
323     case AArch64::ADDSXrr:
324     case AArch64::SUBWrr:
325     case AArch64::SUBXrr:
326     case AArch64::SUBSWrr:
327     case AArch64::SUBSXrr:
328       return true;
329     case AArch64::ADDWrs:
330     case AArch64::ADDXrs:
331     case AArch64::ADDSWrs:
332     case AArch64::ADDSXrs:
333     case AArch64::SUBWrs:
334     case AArch64::SUBXrs:
335     case AArch64::SUBSWrs:
336     case AArch64::SUBSXrs:
337       return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
338     }
339     break;
340 
341   // Arithmetic, setting flags.
342   case AArch64::ADDSWrr:
343   case AArch64::ADDSXrr:
344   case AArch64::SUBSWrr:
345   case AArch64::SUBSXrr:
346   case AArch64::ADDSWrs:
347   case AArch64::ADDSXrs:
348   case AArch64::SUBSWrs:
349   case AArch64::SUBSXrs:
350     // Assume the 1st instr to be a wildcard if it is unspecified.
351     if (FirstMI == nullptr)
352       return true;
353 
354     // Arithmetic, not setting flags.
355     switch (FirstMI->getOpcode()) {
356     case AArch64::ADDWrr:
357     case AArch64::ADDXrr:
358     case AArch64::SUBWrr:
359     case AArch64::SUBXrr:
360       return true;
361     case AArch64::ADDWrs:
362     case AArch64::ADDXrs:
363     case AArch64::SUBWrs:
364     case AArch64::SUBXrs:
365       return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
366     }
367     break;
368   }
369 
370   return false;
371 }
372 
373 /// \brief Check if the instr pair, FirstMI and SecondMI, should be fused
374 /// together. Given SecondMI, when FirstMI is unspecified, then check if
375 /// SecondMI may be part of a fused pair at all.
shouldScheduleAdjacent(const TargetInstrInfo & TII,const TargetSubtargetInfo & TSI,const MachineInstr * FirstMI,const MachineInstr & SecondMI)376 static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
377                                    const TargetSubtargetInfo &TSI,
378                                    const MachineInstr *FirstMI,
379                                    const MachineInstr &SecondMI) {
380   const AArch64Subtarget &ST = static_cast<const AArch64Subtarget&>(TSI);
381 
382   // All checking functions assume that the 1st instr is a wildcard if it is
383   // unspecified.
384   if (ST.hasArithmeticBccFusion() && isArithmeticBccPair(FirstMI, SecondMI))
385     return true;
386   if (ST.hasArithmeticCbzFusion() && isArithmeticCbzPair(FirstMI, SecondMI))
387     return true;
388   if (ST.hasFuseAES() && isAESPair(FirstMI, SecondMI))
389     return true;
390   if (ST.hasFuseCryptoEOR() && isCryptoEORPair(FirstMI, SecondMI))
391     return true;
392   if (ST.hasFuseLiterals() && isLiteralsPair(FirstMI, SecondMI))
393     return true;
394   if (ST.hasFuseAddress() && isAddressLdStPair(FirstMI, SecondMI))
395     return true;
396   if (ST.hasFuseCCSelect() && isCCSelectPair(FirstMI, SecondMI))
397     return true;
398   if (ST.hasFuseArithmeticLogic() && isArithmeticLogicPair(FirstMI, SecondMI))
399     return true;
400 
401   return false;
402 }
403 
404 } // end namespace
405 
406 
407 namespace llvm {
408 
createAArch64MacroFusionDAGMutation()409 std::unique_ptr<ScheduleDAGMutation> createAArch64MacroFusionDAGMutation () {
410   return createMacroFusionDAGMutation(shouldScheduleAdjacent);
411 }
412 
413 } // end namespace llvm
414