1 //===- AArch64RegisterBankInfo.cpp ----------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 /// This file implements the targeting of the RegisterBankInfo class for
10 /// AArch64.
11 /// \todo This should be generated by TableGen.
12 //===----------------------------------------------------------------------===//
13
14 #include "AArch64RegisterBankInfo.h"
15 #include "AArch64RegisterInfo.h"
16 #include "MCTargetDesc/AArch64MCTargetDesc.h"
17 #include "llvm/ADT/STLExtras.h"
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
20 #include "llvm/CodeGen/GlobalISel/Utils.h"
21 #include "llvm/CodeGen/LowLevelType.h"
22 #include "llvm/CodeGen/MachineFunction.h"
23 #include "llvm/CodeGen/MachineInstr.h"
24 #include "llvm/CodeGen/MachineOperand.h"
25 #include "llvm/CodeGen/MachineRegisterInfo.h"
26 #include "llvm/CodeGen/RegisterBank.h"
27 #include "llvm/CodeGen/RegisterBankInfo.h"
28 #include "llvm/CodeGen/TargetOpcodes.h"
29 #include "llvm/CodeGen/TargetRegisterInfo.h"
30 #include "llvm/CodeGen/TargetSubtargetInfo.h"
31 #include "llvm/IR/IntrinsicsAArch64.h"
32 #include "llvm/Support/ErrorHandling.h"
33 #include <algorithm>
34 #include <cassert>
35
36 #define GET_TARGET_REGBANK_IMPL
37 #include "AArch64GenRegisterBank.inc"
38
39 // This file will be TableGen'ed at some point.
40 #include "AArch64GenRegisterBankInfo.def"
41
42 using namespace llvm;
43
AArch64RegisterBankInfo(const TargetRegisterInfo & TRI)44 AArch64RegisterBankInfo::AArch64RegisterBankInfo(
45 const TargetRegisterInfo &TRI) {
46 static llvm::once_flag InitializeRegisterBankFlag;
47
48 static auto InitializeRegisterBankOnce = [&]() {
49 // We have only one set of register banks, whatever the subtarget
50 // is. Therefore, the initialization of the RegBanks table should be
51 // done only once. Indeed the table of all register banks
52 // (AArch64::RegBanks) is unique in the compiler. At some point, it
53 // will get tablegen'ed and the whole constructor becomes empty.
54
55 const RegisterBank &RBGPR = getRegBank(AArch64::GPRRegBankID);
56 (void)RBGPR;
57 assert(&AArch64::GPRRegBank == &RBGPR &&
58 "The order in RegBanks is messed up");
59
60 const RegisterBank &RBFPR = getRegBank(AArch64::FPRRegBankID);
61 (void)RBFPR;
62 assert(&AArch64::FPRRegBank == &RBFPR &&
63 "The order in RegBanks is messed up");
64
65 const RegisterBank &RBCCR = getRegBank(AArch64::CCRegBankID);
66 (void)RBCCR;
67 assert(&AArch64::CCRegBank == &RBCCR &&
68 "The order in RegBanks is messed up");
69
70 // The GPR register bank is fully defined by all the registers in
71 // GR64all + its subclasses.
72 assert(RBGPR.covers(*TRI.getRegClass(AArch64::GPR32RegClassID)) &&
73 "Subclass not added?");
74 assert(RBGPR.getSize() == 128 && "GPRs should hold up to 128-bit");
75
76 // The FPR register bank is fully defined by all the registers in
77 // GR64all + its subclasses.
78 assert(RBFPR.covers(*TRI.getRegClass(AArch64::QQRegClassID)) &&
79 "Subclass not added?");
80 assert(RBFPR.covers(*TRI.getRegClass(AArch64::FPR64RegClassID)) &&
81 "Subclass not added?");
82 assert(RBFPR.getSize() == 512 &&
83 "FPRs should hold up to 512-bit via QQQQ sequence");
84
85 assert(RBCCR.covers(*TRI.getRegClass(AArch64::CCRRegClassID)) &&
86 "Class not added?");
87 assert(RBCCR.getSize() == 32 && "CCR should hold up to 32-bit");
88
89 // Check that the TableGen'ed like file is in sync we our expectations.
90 // First, the Idx.
91 assert(checkPartialMappingIdx(PMI_FirstGPR, PMI_LastGPR,
92 {PMI_GPR32, PMI_GPR64, PMI_GPR128}) &&
93 "PartialMappingIdx's are incorrectly ordered");
94 assert(checkPartialMappingIdx(PMI_FirstFPR, PMI_LastFPR,
95 {PMI_FPR16, PMI_FPR32, PMI_FPR64, PMI_FPR128,
96 PMI_FPR256, PMI_FPR512}) &&
97 "PartialMappingIdx's are incorrectly ordered");
98 // Now, the content.
99 // Check partial mapping.
100 #define CHECK_PARTIALMAP(Idx, ValStartIdx, ValLength, RB) \
101 do { \
102 assert( \
103 checkPartialMap(PartialMappingIdx::Idx, ValStartIdx, ValLength, RB) && \
104 #Idx " is incorrectly initialized"); \
105 } while (false)
106
107 CHECK_PARTIALMAP(PMI_GPR32, 0, 32, RBGPR);
108 CHECK_PARTIALMAP(PMI_GPR64, 0, 64, RBGPR);
109 CHECK_PARTIALMAP(PMI_GPR128, 0, 128, RBGPR);
110 CHECK_PARTIALMAP(PMI_FPR16, 0, 16, RBFPR);
111 CHECK_PARTIALMAP(PMI_FPR32, 0, 32, RBFPR);
112 CHECK_PARTIALMAP(PMI_FPR64, 0, 64, RBFPR);
113 CHECK_PARTIALMAP(PMI_FPR128, 0, 128, RBFPR);
114 CHECK_PARTIALMAP(PMI_FPR256, 0, 256, RBFPR);
115 CHECK_PARTIALMAP(PMI_FPR512, 0, 512, RBFPR);
116
117 // Check value mapping.
118 #define CHECK_VALUEMAP_IMPL(RBName, Size, Offset) \
119 do { \
120 assert(checkValueMapImpl(PartialMappingIdx::PMI_##RBName##Size, \
121 PartialMappingIdx::PMI_First##RBName, Size, \
122 Offset) && \
123 #RBName #Size " " #Offset " is incorrectly initialized"); \
124 } while (false)
125
126 #define CHECK_VALUEMAP(RBName, Size) CHECK_VALUEMAP_IMPL(RBName, Size, 0)
127
128 CHECK_VALUEMAP(GPR, 32);
129 CHECK_VALUEMAP(GPR, 64);
130 CHECK_VALUEMAP(GPR, 128);
131 CHECK_VALUEMAP(FPR, 16);
132 CHECK_VALUEMAP(FPR, 32);
133 CHECK_VALUEMAP(FPR, 64);
134 CHECK_VALUEMAP(FPR, 128);
135 CHECK_VALUEMAP(FPR, 256);
136 CHECK_VALUEMAP(FPR, 512);
137
138 // Check the value mapping for 3-operands instructions where all the operands
139 // map to the same value mapping.
140 #define CHECK_VALUEMAP_3OPS(RBName, Size) \
141 do { \
142 CHECK_VALUEMAP_IMPL(RBName, Size, 0); \
143 CHECK_VALUEMAP_IMPL(RBName, Size, 1); \
144 CHECK_VALUEMAP_IMPL(RBName, Size, 2); \
145 } while (false)
146
147 CHECK_VALUEMAP_3OPS(GPR, 32);
148 CHECK_VALUEMAP_3OPS(GPR, 64);
149 CHECK_VALUEMAP_3OPS(GPR, 128);
150 CHECK_VALUEMAP_3OPS(FPR, 32);
151 CHECK_VALUEMAP_3OPS(FPR, 64);
152 CHECK_VALUEMAP_3OPS(FPR, 128);
153 CHECK_VALUEMAP_3OPS(FPR, 256);
154 CHECK_VALUEMAP_3OPS(FPR, 512);
155
156 #define CHECK_VALUEMAP_CROSSREGCPY(RBNameDst, RBNameSrc, Size) \
157 do { \
158 unsigned PartialMapDstIdx = PMI_##RBNameDst##Size - PMI_Min; \
159 unsigned PartialMapSrcIdx = PMI_##RBNameSrc##Size - PMI_Min; \
160 (void)PartialMapDstIdx; \
161 (void)PartialMapSrcIdx; \
162 const ValueMapping *Map = getCopyMapping( \
163 AArch64::RBNameDst##RegBankID, AArch64::RBNameSrc##RegBankID, Size); \
164 (void)Map; \
165 assert(Map[0].BreakDown == \
166 &AArch64GenRegisterBankInfo::PartMappings[PartialMapDstIdx] && \
167 Map[0].NumBreakDowns == 1 && #RBNameDst #Size \
168 " Dst is incorrectly initialized"); \
169 assert(Map[1].BreakDown == \
170 &AArch64GenRegisterBankInfo::PartMappings[PartialMapSrcIdx] && \
171 Map[1].NumBreakDowns == 1 && #RBNameSrc #Size \
172 " Src is incorrectly initialized"); \
173 \
174 } while (false)
175
176 CHECK_VALUEMAP_CROSSREGCPY(GPR, GPR, 32);
177 CHECK_VALUEMAP_CROSSREGCPY(GPR, FPR, 32);
178 CHECK_VALUEMAP_CROSSREGCPY(GPR, GPR, 64);
179 CHECK_VALUEMAP_CROSSREGCPY(GPR, FPR, 64);
180 CHECK_VALUEMAP_CROSSREGCPY(FPR, FPR, 32);
181 CHECK_VALUEMAP_CROSSREGCPY(FPR, GPR, 32);
182 CHECK_VALUEMAP_CROSSREGCPY(FPR, FPR, 64);
183 CHECK_VALUEMAP_CROSSREGCPY(FPR, GPR, 64);
184
185 #define CHECK_VALUEMAP_FPEXT(DstSize, SrcSize) \
186 do { \
187 unsigned PartialMapDstIdx = PMI_FPR##DstSize - PMI_Min; \
188 unsigned PartialMapSrcIdx = PMI_FPR##SrcSize - PMI_Min; \
189 (void)PartialMapDstIdx; \
190 (void)PartialMapSrcIdx; \
191 const ValueMapping *Map = getFPExtMapping(DstSize, SrcSize); \
192 (void)Map; \
193 assert(Map[0].BreakDown == \
194 &AArch64GenRegisterBankInfo::PartMappings[PartialMapDstIdx] && \
195 Map[0].NumBreakDowns == 1 && "FPR" #DstSize \
196 " Dst is incorrectly initialized"); \
197 assert(Map[1].BreakDown == \
198 &AArch64GenRegisterBankInfo::PartMappings[PartialMapSrcIdx] && \
199 Map[1].NumBreakDowns == 1 && "FPR" #SrcSize \
200 " Src is incorrectly initialized"); \
201 \
202 } while (false)
203
204 CHECK_VALUEMAP_FPEXT(32, 16);
205 CHECK_VALUEMAP_FPEXT(64, 16);
206 CHECK_VALUEMAP_FPEXT(64, 32);
207 CHECK_VALUEMAP_FPEXT(128, 64);
208
209 assert(verify(TRI) && "Invalid register bank information");
210 };
211
212 llvm::call_once(InitializeRegisterBankFlag, InitializeRegisterBankOnce);
213 }
214
copyCost(const RegisterBank & A,const RegisterBank & B,unsigned Size) const215 unsigned AArch64RegisterBankInfo::copyCost(const RegisterBank &A,
216 const RegisterBank &B,
217 unsigned Size) const {
218 // What do we do with different size?
219 // copy are same size.
220 // Will introduce other hooks for different size:
221 // * extract cost.
222 // * build_sequence cost.
223
224 // Copy from (resp. to) GPR to (resp. from) FPR involves FMOV.
225 // FIXME: This should be deduced from the scheduling model.
226 if (&A == &AArch64::GPRRegBank && &B == &AArch64::FPRRegBank)
227 // FMOVXDr or FMOVWSr.
228 return 5;
229 if (&A == &AArch64::FPRRegBank && &B == &AArch64::GPRRegBank)
230 // FMOVDXr or FMOVSWr.
231 return 4;
232
233 return RegisterBankInfo::copyCost(A, B, Size);
234 }
235
236 const RegisterBank &
getRegBankFromRegClass(const TargetRegisterClass & RC,LLT) const237 AArch64RegisterBankInfo::getRegBankFromRegClass(const TargetRegisterClass &RC,
238 LLT) const {
239 switch (RC.getID()) {
240 case AArch64::FPR8RegClassID:
241 case AArch64::FPR16RegClassID:
242 case AArch64::FPR16_loRegClassID:
243 case AArch64::FPR32_with_hsub_in_FPR16_loRegClassID:
244 case AArch64::FPR32RegClassID:
245 case AArch64::FPR64RegClassID:
246 case AArch64::FPR64_loRegClassID:
247 case AArch64::FPR128RegClassID:
248 case AArch64::FPR128_loRegClassID:
249 case AArch64::DDRegClassID:
250 case AArch64::DDDRegClassID:
251 case AArch64::DDDDRegClassID:
252 case AArch64::QQRegClassID:
253 case AArch64::QQQRegClassID:
254 case AArch64::QQQQRegClassID:
255 return getRegBank(AArch64::FPRRegBankID);
256 case AArch64::GPR32commonRegClassID:
257 case AArch64::GPR32RegClassID:
258 case AArch64::GPR32spRegClassID:
259 case AArch64::GPR32sponlyRegClassID:
260 case AArch64::GPR32argRegClassID:
261 case AArch64::GPR32allRegClassID:
262 case AArch64::GPR64commonRegClassID:
263 case AArch64::GPR64RegClassID:
264 case AArch64::GPR64spRegClassID:
265 case AArch64::GPR64sponlyRegClassID:
266 case AArch64::GPR64argRegClassID:
267 case AArch64::GPR64allRegClassID:
268 case AArch64::GPR64noipRegClassID:
269 case AArch64::GPR64common_and_GPR64noipRegClassID:
270 case AArch64::GPR64noip_and_tcGPR64RegClassID:
271 case AArch64::tcGPR64RegClassID:
272 case AArch64::rtcGPR64RegClassID:
273 case AArch64::WSeqPairsClassRegClassID:
274 case AArch64::XSeqPairsClassRegClassID:
275 case AArch64::MatrixIndexGPR32_12_15RegClassID:
276 case AArch64::GPR64_with_sub_32_in_MatrixIndexGPR32_12_15RegClassID:
277 return getRegBank(AArch64::GPRRegBankID);
278 case AArch64::CCRRegClassID:
279 return getRegBank(AArch64::CCRegBankID);
280 default:
281 llvm_unreachable("Register class not supported");
282 }
283 }
284
285 RegisterBankInfo::InstructionMappings
getInstrAlternativeMappings(const MachineInstr & MI) const286 AArch64RegisterBankInfo::getInstrAlternativeMappings(
287 const MachineInstr &MI) const {
288 const MachineFunction &MF = *MI.getParent()->getParent();
289 const TargetSubtargetInfo &STI = MF.getSubtarget();
290 const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
291 const MachineRegisterInfo &MRI = MF.getRegInfo();
292
293 switch (MI.getOpcode()) {
294 case TargetOpcode::G_OR: {
295 // 32 and 64-bit or can be mapped on either FPR or
296 // GPR for the same cost.
297 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, TRI);
298 if (Size != 32 && Size != 64)
299 break;
300
301 // If the instruction has any implicit-defs or uses,
302 // do not mess with it.
303 if (MI.getNumOperands() != 3)
304 break;
305 InstructionMappings AltMappings;
306 const InstructionMapping &GPRMapping = getInstructionMapping(
307 /*ID*/ 1, /*Cost*/ 1, getValueMapping(PMI_FirstGPR, Size),
308 /*NumOperands*/ 3);
309 const InstructionMapping &FPRMapping = getInstructionMapping(
310 /*ID*/ 2, /*Cost*/ 1, getValueMapping(PMI_FirstFPR, Size),
311 /*NumOperands*/ 3);
312
313 AltMappings.push_back(&GPRMapping);
314 AltMappings.push_back(&FPRMapping);
315 return AltMappings;
316 }
317 case TargetOpcode::G_BITCAST: {
318 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, TRI);
319 if (Size != 32 && Size != 64)
320 break;
321
322 // If the instruction has any implicit-defs or uses,
323 // do not mess with it.
324 if (MI.getNumOperands() != 2)
325 break;
326
327 InstructionMappings AltMappings;
328 const InstructionMapping &GPRMapping = getInstructionMapping(
329 /*ID*/ 1, /*Cost*/ 1,
330 getCopyMapping(AArch64::GPRRegBankID, AArch64::GPRRegBankID, Size),
331 /*NumOperands*/ 2);
332 const InstructionMapping &FPRMapping = getInstructionMapping(
333 /*ID*/ 2, /*Cost*/ 1,
334 getCopyMapping(AArch64::FPRRegBankID, AArch64::FPRRegBankID, Size),
335 /*NumOperands*/ 2);
336 const InstructionMapping &GPRToFPRMapping = getInstructionMapping(
337 /*ID*/ 3,
338 /*Cost*/ copyCost(AArch64::GPRRegBank, AArch64::FPRRegBank, Size),
339 getCopyMapping(AArch64::FPRRegBankID, AArch64::GPRRegBankID, Size),
340 /*NumOperands*/ 2);
341 const InstructionMapping &FPRToGPRMapping = getInstructionMapping(
342 /*ID*/ 3,
343 /*Cost*/ copyCost(AArch64::GPRRegBank, AArch64::FPRRegBank, Size),
344 getCopyMapping(AArch64::GPRRegBankID, AArch64::FPRRegBankID, Size),
345 /*NumOperands*/ 2);
346
347 AltMappings.push_back(&GPRMapping);
348 AltMappings.push_back(&FPRMapping);
349 AltMappings.push_back(&GPRToFPRMapping);
350 AltMappings.push_back(&FPRToGPRMapping);
351 return AltMappings;
352 }
353 case TargetOpcode::G_LOAD: {
354 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, TRI);
355 if (Size != 64)
356 break;
357
358 // If the instruction has any implicit-defs or uses,
359 // do not mess with it.
360 if (MI.getNumOperands() != 2)
361 break;
362
363 InstructionMappings AltMappings;
364 const InstructionMapping &GPRMapping = getInstructionMapping(
365 /*ID*/ 1, /*Cost*/ 1,
366 getOperandsMapping({getValueMapping(PMI_FirstGPR, Size),
367 // Addresses are GPR 64-bit.
368 getValueMapping(PMI_FirstGPR, 64)}),
369 /*NumOperands*/ 2);
370 const InstructionMapping &FPRMapping = getInstructionMapping(
371 /*ID*/ 2, /*Cost*/ 1,
372 getOperandsMapping({getValueMapping(PMI_FirstFPR, Size),
373 // Addresses are GPR 64-bit.
374 getValueMapping(PMI_FirstGPR, 64)}),
375 /*NumOperands*/ 2);
376
377 AltMappings.push_back(&GPRMapping);
378 AltMappings.push_back(&FPRMapping);
379 return AltMappings;
380 }
381 default:
382 break;
383 }
384 return RegisterBankInfo::getInstrAlternativeMappings(MI);
385 }
386
applyMappingImpl(const OperandsMapper & OpdMapper) const387 void AArch64RegisterBankInfo::applyMappingImpl(
388 const OperandsMapper &OpdMapper) const {
389 switch (OpdMapper.getMI().getOpcode()) {
390 case TargetOpcode::G_OR:
391 case TargetOpcode::G_BITCAST:
392 case TargetOpcode::G_LOAD:
393 // Those ID must match getInstrAlternativeMappings.
394 assert((OpdMapper.getInstrMapping().getID() >= 1 &&
395 OpdMapper.getInstrMapping().getID() <= 4) &&
396 "Don't know how to handle that ID");
397 return applyDefaultMapping(OpdMapper);
398 default:
399 llvm_unreachable("Don't know how to handle that operation");
400 }
401 }
402
403 /// Returns whether opcode \p Opc is a pre-isel generic floating-point opcode,
404 /// having only floating-point operands.
isPreISelGenericFloatingPointOpcode(unsigned Opc)405 static bool isPreISelGenericFloatingPointOpcode(unsigned Opc) {
406 switch (Opc) {
407 case TargetOpcode::G_FADD:
408 case TargetOpcode::G_FSUB:
409 case TargetOpcode::G_FMUL:
410 case TargetOpcode::G_FMA:
411 case TargetOpcode::G_FDIV:
412 case TargetOpcode::G_FCONSTANT:
413 case TargetOpcode::G_FPEXT:
414 case TargetOpcode::G_FPTRUNC:
415 case TargetOpcode::G_FCEIL:
416 case TargetOpcode::G_FFLOOR:
417 case TargetOpcode::G_FNEARBYINT:
418 case TargetOpcode::G_FNEG:
419 case TargetOpcode::G_FCOS:
420 case TargetOpcode::G_FSIN:
421 case TargetOpcode::G_FLOG10:
422 case TargetOpcode::G_FLOG:
423 case TargetOpcode::G_FLOG2:
424 case TargetOpcode::G_FSQRT:
425 case TargetOpcode::G_FABS:
426 case TargetOpcode::G_FEXP:
427 case TargetOpcode::G_FRINT:
428 case TargetOpcode::G_INTRINSIC_TRUNC:
429 case TargetOpcode::G_INTRINSIC_ROUND:
430 case TargetOpcode::G_FMAXNUM:
431 case TargetOpcode::G_FMINNUM:
432 case TargetOpcode::G_FMAXIMUM:
433 case TargetOpcode::G_FMINIMUM:
434 return true;
435 }
436 return false;
437 }
438
439 const RegisterBankInfo::InstructionMapping &
getSameKindOfOperandsMapping(const MachineInstr & MI) const440 AArch64RegisterBankInfo::getSameKindOfOperandsMapping(
441 const MachineInstr &MI) const {
442 const unsigned Opc = MI.getOpcode();
443 const MachineFunction &MF = *MI.getParent()->getParent();
444 const MachineRegisterInfo &MRI = MF.getRegInfo();
445
446 unsigned NumOperands = MI.getNumOperands();
447 assert(NumOperands <= 3 &&
448 "This code is for instructions with 3 or less operands");
449
450 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
451 unsigned Size = Ty.getSizeInBits();
452 bool IsFPR = Ty.isVector() || isPreISelGenericFloatingPointOpcode(Opc);
453
454 PartialMappingIdx RBIdx = IsFPR ? PMI_FirstFPR : PMI_FirstGPR;
455
456 #ifndef NDEBUG
457 // Make sure all the operands are using similar size and type.
458 // Should probably be checked by the machine verifier.
459 // This code won't catch cases where the number of lanes is
460 // different between the operands.
461 // If we want to go to that level of details, it is probably
462 // best to check that the types are the same, period.
463 // Currently, we just check that the register banks are the same
464 // for each types.
465 for (unsigned Idx = 1; Idx != NumOperands; ++Idx) {
466 LLT OpTy = MRI.getType(MI.getOperand(Idx).getReg());
467 assert(
468 AArch64GenRegisterBankInfo::getRegBankBaseIdxOffset(
469 RBIdx, OpTy.getSizeInBits()) ==
470 AArch64GenRegisterBankInfo::getRegBankBaseIdxOffset(RBIdx, Size) &&
471 "Operand has incompatible size");
472 bool OpIsFPR = OpTy.isVector() || isPreISelGenericFloatingPointOpcode(Opc);
473 (void)OpIsFPR;
474 assert(IsFPR == OpIsFPR && "Operand has incompatible type");
475 }
476 #endif // End NDEBUG.
477
478 return getInstructionMapping(DefaultMappingID, 1,
479 getValueMapping(RBIdx, Size), NumOperands);
480 }
481
482 /// \returns true if a given intrinsic \p ID only uses and defines FPRs.
isFPIntrinsic(unsigned ID)483 static bool isFPIntrinsic(unsigned ID) {
484 // TODO: Add more intrinsics.
485 switch (ID) {
486 default:
487 return false;
488 case Intrinsic::aarch64_neon_uaddlv:
489 return true;
490 }
491 }
492
hasFPConstraints(const MachineInstr & MI,const MachineRegisterInfo & MRI,const TargetRegisterInfo & TRI,unsigned Depth) const493 bool AArch64RegisterBankInfo::hasFPConstraints(const MachineInstr &MI,
494 const MachineRegisterInfo &MRI,
495 const TargetRegisterInfo &TRI,
496 unsigned Depth) const {
497 unsigned Op = MI.getOpcode();
498 if (Op == TargetOpcode::G_INTRINSIC && isFPIntrinsic(MI.getIntrinsicID()))
499 return true;
500
501 // Do we have an explicit floating point instruction?
502 if (isPreISelGenericFloatingPointOpcode(Op))
503 return true;
504
505 // No. Check if we have a copy-like instruction. If we do, then we could
506 // still be fed by floating point instructions.
507 if (Op != TargetOpcode::COPY && !MI.isPHI() &&
508 !isPreISelGenericOptimizationHint(Op))
509 return false;
510
511 // Check if we already know the register bank.
512 auto *RB = getRegBank(MI.getOperand(0).getReg(), MRI, TRI);
513 if (RB == &AArch64::FPRRegBank)
514 return true;
515 if (RB == &AArch64::GPRRegBank)
516 return false;
517
518 // We don't know anything.
519 //
520 // If we have a phi, we may be able to infer that it will be assigned a FPR
521 // based off of its inputs.
522 if (!MI.isPHI() || Depth > MaxFPRSearchDepth)
523 return false;
524
525 return any_of(MI.explicit_uses(), [&](const MachineOperand &Op) {
526 return Op.isReg() &&
527 onlyDefinesFP(*MRI.getVRegDef(Op.getReg()), MRI, TRI, Depth + 1);
528 });
529 }
530
onlyUsesFP(const MachineInstr & MI,const MachineRegisterInfo & MRI,const TargetRegisterInfo & TRI,unsigned Depth) const531 bool AArch64RegisterBankInfo::onlyUsesFP(const MachineInstr &MI,
532 const MachineRegisterInfo &MRI,
533 const TargetRegisterInfo &TRI,
534 unsigned Depth) const {
535 switch (MI.getOpcode()) {
536 case TargetOpcode::G_FPTOSI:
537 case TargetOpcode::G_FPTOUI:
538 case TargetOpcode::G_FCMP:
539 case TargetOpcode::G_LROUND:
540 case TargetOpcode::G_LLROUND:
541 return true;
542 default:
543 break;
544 }
545 return hasFPConstraints(MI, MRI, TRI, Depth);
546 }
547
onlyDefinesFP(const MachineInstr & MI,const MachineRegisterInfo & MRI,const TargetRegisterInfo & TRI,unsigned Depth) const548 bool AArch64RegisterBankInfo::onlyDefinesFP(const MachineInstr &MI,
549 const MachineRegisterInfo &MRI,
550 const TargetRegisterInfo &TRI,
551 unsigned Depth) const {
552 switch (MI.getOpcode()) {
553 case AArch64::G_DUP:
554 case TargetOpcode::G_SITOFP:
555 case TargetOpcode::G_UITOFP:
556 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
557 case TargetOpcode::G_INSERT_VECTOR_ELT:
558 case TargetOpcode::G_BUILD_VECTOR:
559 case TargetOpcode::G_BUILD_VECTOR_TRUNC:
560 return true;
561 default:
562 break;
563 }
564 return hasFPConstraints(MI, MRI, TRI, Depth);
565 }
566
567 const RegisterBankInfo::InstructionMapping &
getInstrMapping(const MachineInstr & MI) const568 AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
569 const unsigned Opc = MI.getOpcode();
570
571 // Try the default logic for non-generic instructions that are either copies
572 // or already have some operands assigned to banks.
573 if ((Opc != TargetOpcode::COPY && !isPreISelGenericOpcode(Opc)) ||
574 Opc == TargetOpcode::G_PHI) {
575 const RegisterBankInfo::InstructionMapping &Mapping =
576 getInstrMappingImpl(MI);
577 if (Mapping.isValid())
578 return Mapping;
579 }
580
581 const MachineFunction &MF = *MI.getParent()->getParent();
582 const MachineRegisterInfo &MRI = MF.getRegInfo();
583 const TargetSubtargetInfo &STI = MF.getSubtarget();
584 const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
585
586 switch (Opc) {
587 // G_{F|S|U}REM are not listed because they are not legal.
588 // Arithmetic ops.
589 case TargetOpcode::G_ADD:
590 case TargetOpcode::G_SUB:
591 case TargetOpcode::G_PTR_ADD:
592 case TargetOpcode::G_MUL:
593 case TargetOpcode::G_SDIV:
594 case TargetOpcode::G_UDIV:
595 // Bitwise ops.
596 case TargetOpcode::G_AND:
597 case TargetOpcode::G_OR:
598 case TargetOpcode::G_XOR:
599 // Floating point ops.
600 case TargetOpcode::G_FADD:
601 case TargetOpcode::G_FSUB:
602 case TargetOpcode::G_FMUL:
603 case TargetOpcode::G_FDIV:
604 case TargetOpcode::G_FMAXIMUM:
605 case TargetOpcode::G_FMINIMUM:
606 return getSameKindOfOperandsMapping(MI);
607 case TargetOpcode::G_FPEXT: {
608 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
609 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
610 return getInstructionMapping(
611 DefaultMappingID, /*Cost*/ 1,
612 getFPExtMapping(DstTy.getSizeInBits(), SrcTy.getSizeInBits()),
613 /*NumOperands*/ 2);
614 }
615 // Shifts.
616 case TargetOpcode::G_SHL:
617 case TargetOpcode::G_LSHR:
618 case TargetOpcode::G_ASHR: {
619 LLT ShiftAmtTy = MRI.getType(MI.getOperand(2).getReg());
620 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
621 if (ShiftAmtTy.getSizeInBits() == 64 && SrcTy.getSizeInBits() == 32)
622 return getInstructionMapping(DefaultMappingID, 1,
623 &ValMappings[Shift64Imm], 3);
624 return getSameKindOfOperandsMapping(MI);
625 }
626 case TargetOpcode::COPY: {
627 Register DstReg = MI.getOperand(0).getReg();
628 Register SrcReg = MI.getOperand(1).getReg();
629 // Check if one of the register is not a generic register.
630 if ((Register::isPhysicalRegister(DstReg) ||
631 !MRI.getType(DstReg).isValid()) ||
632 (Register::isPhysicalRegister(SrcReg) ||
633 !MRI.getType(SrcReg).isValid())) {
634 const RegisterBank *DstRB = getRegBank(DstReg, MRI, TRI);
635 const RegisterBank *SrcRB = getRegBank(SrcReg, MRI, TRI);
636 if (!DstRB)
637 DstRB = SrcRB;
638 else if (!SrcRB)
639 SrcRB = DstRB;
640 // If both RB are null that means both registers are generic.
641 // We shouldn't be here.
642 assert(DstRB && SrcRB && "Both RegBank were nullptr");
643 unsigned Size = getSizeInBits(DstReg, MRI, TRI);
644 return getInstructionMapping(
645 DefaultMappingID, copyCost(*DstRB, *SrcRB, Size),
646 getCopyMapping(DstRB->getID(), SrcRB->getID(), Size),
647 // We only care about the mapping of the destination.
648 /*NumOperands*/ 1);
649 }
650 // Both registers are generic, use G_BITCAST.
651 LLVM_FALLTHROUGH;
652 }
653 case TargetOpcode::G_BITCAST: {
654 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
655 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
656 unsigned Size = DstTy.getSizeInBits();
657 bool DstIsGPR = !DstTy.isVector() && DstTy.getSizeInBits() <= 64;
658 bool SrcIsGPR = !SrcTy.isVector() && SrcTy.getSizeInBits() <= 64;
659 const RegisterBank &DstRB =
660 DstIsGPR ? AArch64::GPRRegBank : AArch64::FPRRegBank;
661 const RegisterBank &SrcRB =
662 SrcIsGPR ? AArch64::GPRRegBank : AArch64::FPRRegBank;
663 return getInstructionMapping(
664 DefaultMappingID, copyCost(DstRB, SrcRB, Size),
665 getCopyMapping(DstRB.getID(), SrcRB.getID(), Size),
666 // We only care about the mapping of the destination for COPY.
667 /*NumOperands*/ Opc == TargetOpcode::G_BITCAST ? 2 : 1);
668 }
669 default:
670 break;
671 }
672
673 unsigned NumOperands = MI.getNumOperands();
674
675 // Track the size and bank of each register. We don't do partial mappings.
676 SmallVector<unsigned, 4> OpSize(NumOperands);
677 SmallVector<PartialMappingIdx, 4> OpRegBankIdx(NumOperands);
678 for (unsigned Idx = 0; Idx < NumOperands; ++Idx) {
679 auto &MO = MI.getOperand(Idx);
680 if (!MO.isReg() || !MO.getReg())
681 continue;
682
683 LLT Ty = MRI.getType(MO.getReg());
684 OpSize[Idx] = Ty.getSizeInBits();
685
686 // As a top-level guess, vectors go in FPRs, scalars and pointers in GPRs.
687 // For floating-point instructions, scalars go in FPRs.
688 if (Ty.isVector() || isPreISelGenericFloatingPointOpcode(Opc) ||
689 Ty.getSizeInBits() > 64)
690 OpRegBankIdx[Idx] = PMI_FirstFPR;
691 else
692 OpRegBankIdx[Idx] = PMI_FirstGPR;
693 }
694
695 unsigned Cost = 1;
696 // Some of the floating-point instructions have mixed GPR and FPR operands:
697 // fine-tune the computed mapping.
698 switch (Opc) {
699 case AArch64::G_DUP: {
700 Register ScalarReg = MI.getOperand(1).getReg();
701 LLT ScalarTy = MRI.getType(ScalarReg);
702 auto ScalarDef = MRI.getVRegDef(ScalarReg);
703 // s8 is an exception for G_DUP, which we always want on gpr.
704 if (ScalarTy.getSizeInBits() != 8 &&
705 (getRegBank(ScalarReg, MRI, TRI) == &AArch64::FPRRegBank ||
706 onlyDefinesFP(*ScalarDef, MRI, TRI)))
707 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
708 else
709 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR};
710 break;
711 }
712 case TargetOpcode::G_TRUNC: {
713 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
714 if (!SrcTy.isVector() && SrcTy.getSizeInBits() == 128)
715 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
716 break;
717 }
718 case TargetOpcode::G_SITOFP:
719 case TargetOpcode::G_UITOFP: {
720 if (MRI.getType(MI.getOperand(0).getReg()).isVector())
721 break;
722 // Integer to FP conversions don't necessarily happen between GPR -> FPR
723 // regbanks. They can also be done within an FPR register.
724 Register SrcReg = MI.getOperand(1).getReg();
725 if (getRegBank(SrcReg, MRI, TRI) == &AArch64::FPRRegBank)
726 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
727 else
728 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR};
729 break;
730 }
731 case TargetOpcode::G_FPTOSI:
732 case TargetOpcode::G_FPTOUI:
733 if (MRI.getType(MI.getOperand(0).getReg()).isVector())
734 break;
735 OpRegBankIdx = {PMI_FirstGPR, PMI_FirstFPR};
736 break;
737 case TargetOpcode::G_FCMP: {
738 // If the result is a vector, it must use a FPR.
739 AArch64GenRegisterBankInfo::PartialMappingIdx Idx0 =
740 MRI.getType(MI.getOperand(0).getReg()).isVector() ? PMI_FirstFPR
741 : PMI_FirstGPR;
742 OpRegBankIdx = {Idx0,
743 /* Predicate */ PMI_None, PMI_FirstFPR, PMI_FirstFPR};
744 break;
745 }
746 case TargetOpcode::G_BITCAST:
747 // This is going to be a cross register bank copy and this is expensive.
748 if (OpRegBankIdx[0] != OpRegBankIdx[1])
749 Cost = copyCost(
750 *AArch64GenRegisterBankInfo::PartMappings[OpRegBankIdx[0]].RegBank,
751 *AArch64GenRegisterBankInfo::PartMappings[OpRegBankIdx[1]].RegBank,
752 OpSize[0]);
753 break;
754 case TargetOpcode::G_LOAD:
755 // Loading in vector unit is slightly more expensive.
756 // This is actually only true for the LD1R and co instructions,
757 // but anyway for the fast mode this number does not matter and
758 // for the greedy mode the cost of the cross bank copy will
759 // offset this number.
760 // FIXME: Should be derived from the scheduling model.
761 if (OpRegBankIdx[0] != PMI_FirstGPR) {
762 Cost = 2;
763 break;
764 }
765
766 if (cast<GLoad>(MI).isAtomic()) {
767 // Atomics always use GPR destinations. Don't refine any further.
768 OpRegBankIdx[0] = PMI_FirstGPR;
769 break;
770 }
771
772 // Check if that load feeds fp instructions.
773 // In that case, we want the default mapping to be on FPR
774 // instead of blind map every scalar to GPR.
775 if (any_of(MRI.use_nodbg_instructions(MI.getOperand(0).getReg()),
776 [&](const MachineInstr &UseMI) {
777 // If we have at least one direct use in a FP instruction,
778 // assume this was a floating point load in the IR. If it was
779 // not, we would have had a bitcast before reaching that
780 // instruction.
781 //
782 // Int->FP conversion operations are also captured in
783 // onlyDefinesFP().
784 return onlyUsesFP(UseMI, MRI, TRI) ||
785 onlyDefinesFP(UseMI, MRI, TRI);
786 }))
787 OpRegBankIdx[0] = PMI_FirstFPR;
788 break;
789 case TargetOpcode::G_STORE:
790 // Check if that store is fed by fp instructions.
791 if (OpRegBankIdx[0] == PMI_FirstGPR) {
792 Register VReg = MI.getOperand(0).getReg();
793 if (!VReg)
794 break;
795 MachineInstr *DefMI = MRI.getVRegDef(VReg);
796 if (onlyDefinesFP(*DefMI, MRI, TRI))
797 OpRegBankIdx[0] = PMI_FirstFPR;
798 break;
799 }
800 break;
801 case TargetOpcode::G_SELECT: {
802 // If the destination is FPR, preserve that.
803 if (OpRegBankIdx[0] != PMI_FirstGPR)
804 break;
805
806 // If we're taking in vectors, we have no choice but to put everything on
807 // FPRs, except for the condition. The condition must always be on a GPR.
808 LLT SrcTy = MRI.getType(MI.getOperand(2).getReg());
809 if (SrcTy.isVector()) {
810 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR, PMI_FirstFPR, PMI_FirstFPR};
811 break;
812 }
813
814 // Try to minimize the number of copies. If we have more floating point
815 // constrained values than not, then we'll put everything on FPR. Otherwise,
816 // everything has to be on GPR.
817 unsigned NumFP = 0;
818
819 // Check if the uses of the result always produce floating point values.
820 //
821 // For example:
822 //
823 // %z = G_SELECT %cond %x %y
824 // fpr = G_FOO %z ...
825 if (any_of(MRI.use_nodbg_instructions(MI.getOperand(0).getReg()),
826 [&](MachineInstr &MI) { return onlyUsesFP(MI, MRI, TRI); }))
827 ++NumFP;
828
829 // Check if the defs of the source values always produce floating point
830 // values.
831 //
832 // For example:
833 //
834 // %x = G_SOMETHING_ALWAYS_FLOAT %a ...
835 // %z = G_SELECT %cond %x %y
836 //
837 // Also check whether or not the sources have already been decided to be
838 // FPR. Keep track of this.
839 //
840 // This doesn't check the condition, since it's just whatever is in NZCV.
841 // This isn't passed explicitly in a register to fcsel/csel.
842 for (unsigned Idx = 2; Idx < 4; ++Idx) {
843 Register VReg = MI.getOperand(Idx).getReg();
844 MachineInstr *DefMI = MRI.getVRegDef(VReg);
845 if (getRegBank(VReg, MRI, TRI) == &AArch64::FPRRegBank ||
846 onlyDefinesFP(*DefMI, MRI, TRI))
847 ++NumFP;
848 }
849
850 // If we have more FP constraints than not, then move everything over to
851 // FPR.
852 if (NumFP >= 2)
853 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR, PMI_FirstFPR, PMI_FirstFPR};
854
855 break;
856 }
857 case TargetOpcode::G_UNMERGE_VALUES: {
858 // If the first operand belongs to a FPR register bank, then make sure that
859 // we preserve that.
860 if (OpRegBankIdx[0] != PMI_FirstGPR)
861 break;
862
863 LLT SrcTy = MRI.getType(MI.getOperand(MI.getNumOperands()-1).getReg());
864 // UNMERGE into scalars from a vector should always use FPR.
865 // Likewise if any of the uses are FP instructions.
866 if (SrcTy.isVector() || SrcTy == LLT::scalar(128) ||
867 any_of(MRI.use_nodbg_instructions(MI.getOperand(0).getReg()),
868 [&](MachineInstr &MI) { return onlyUsesFP(MI, MRI, TRI); })) {
869 // Set the register bank of every operand to FPR.
870 for (unsigned Idx = 0, NumOperands = MI.getNumOperands();
871 Idx < NumOperands; ++Idx)
872 OpRegBankIdx[Idx] = PMI_FirstFPR;
873 }
874 break;
875 }
876 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
877 // Destination and source need to be FPRs.
878 OpRegBankIdx[0] = PMI_FirstFPR;
879 OpRegBankIdx[1] = PMI_FirstFPR;
880
881 // Index needs to be a GPR.
882 OpRegBankIdx[2] = PMI_FirstGPR;
883 break;
884 case TargetOpcode::G_INSERT_VECTOR_ELT:
885 OpRegBankIdx[0] = PMI_FirstFPR;
886 OpRegBankIdx[1] = PMI_FirstFPR;
887
888 // The element may be either a GPR or FPR. Preserve that behaviour.
889 if (getRegBank(MI.getOperand(2).getReg(), MRI, TRI) == &AArch64::FPRRegBank)
890 OpRegBankIdx[2] = PMI_FirstFPR;
891 else
892 OpRegBankIdx[2] = PMI_FirstGPR;
893
894 // Index needs to be a GPR.
895 OpRegBankIdx[3] = PMI_FirstGPR;
896 break;
897 case TargetOpcode::G_EXTRACT: {
898 // For s128 sources we have to use fpr unless we know otherwise.
899 auto Src = MI.getOperand(1).getReg();
900 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
901 if (SrcTy.getSizeInBits() != 128)
902 break;
903 auto Idx = MRI.getRegClassOrNull(Src) == &AArch64::XSeqPairsClassRegClass
904 ? PMI_FirstGPR
905 : PMI_FirstFPR;
906 OpRegBankIdx[0] = Idx;
907 OpRegBankIdx[1] = Idx;
908 break;
909 }
910 case TargetOpcode::G_BUILD_VECTOR: {
911 // If the first source operand belongs to a FPR register bank, then make
912 // sure that we preserve that.
913 if (OpRegBankIdx[1] != PMI_FirstGPR)
914 break;
915 Register VReg = MI.getOperand(1).getReg();
916 if (!VReg)
917 break;
918
919 // Get the instruction that defined the source operand reg, and check if
920 // it's a floating point operation. Or, if it's a type like s16 which
921 // doesn't have a exact size gpr register class. The exception is if the
922 // build_vector has all constant operands, which may be better to leave as
923 // gpr without copies, so it can be matched in imported patterns.
924 MachineInstr *DefMI = MRI.getVRegDef(VReg);
925 unsigned DefOpc = DefMI->getOpcode();
926 const LLT SrcTy = MRI.getType(VReg);
927 if (all_of(MI.operands(), [&](const MachineOperand &Op) {
928 return Op.isDef() || MRI.getVRegDef(Op.getReg())->getOpcode() ==
929 TargetOpcode::G_CONSTANT;
930 }))
931 break;
932 if (isPreISelGenericFloatingPointOpcode(DefOpc) ||
933 SrcTy.getSizeInBits() < 32 ||
934 getRegBank(VReg, MRI, TRI) == &AArch64::FPRRegBank) {
935 // Have a floating point op.
936 // Make sure every operand gets mapped to a FPR register class.
937 unsigned NumOperands = MI.getNumOperands();
938 for (unsigned Idx = 0; Idx < NumOperands; ++Idx)
939 OpRegBankIdx[Idx] = PMI_FirstFPR;
940 }
941 break;
942 }
943 case TargetOpcode::G_VECREDUCE_FADD:
944 case TargetOpcode::G_VECREDUCE_FMUL:
945 case TargetOpcode::G_VECREDUCE_FMAX:
946 case TargetOpcode::G_VECREDUCE_FMIN:
947 case TargetOpcode::G_VECREDUCE_ADD:
948 case TargetOpcode::G_VECREDUCE_MUL:
949 case TargetOpcode::G_VECREDUCE_AND:
950 case TargetOpcode::G_VECREDUCE_OR:
951 case TargetOpcode::G_VECREDUCE_XOR:
952 case TargetOpcode::G_VECREDUCE_SMAX:
953 case TargetOpcode::G_VECREDUCE_SMIN:
954 case TargetOpcode::G_VECREDUCE_UMAX:
955 case TargetOpcode::G_VECREDUCE_UMIN:
956 // Reductions produce a scalar value from a vector, the scalar should be on
957 // FPR bank.
958 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
959 break;
960 case TargetOpcode::G_VECREDUCE_SEQ_FADD:
961 case TargetOpcode::G_VECREDUCE_SEQ_FMUL:
962 // These reductions also take a scalar accumulator input.
963 // Assign them FPR for now.
964 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR, PMI_FirstFPR};
965 break;
966 case TargetOpcode::G_INTRINSIC: {
967 // Check if we know that the intrinsic has any constraints on its register
968 // banks. If it does, then update the mapping accordingly.
969 unsigned ID = MI.getIntrinsicID();
970 unsigned Idx = 0;
971 if (!isFPIntrinsic(ID))
972 break;
973 for (const auto &Op : MI.explicit_operands()) {
974 if (Op.isReg())
975 OpRegBankIdx[Idx] = PMI_FirstFPR;
976 ++Idx;
977 }
978 break;
979 }
980 case TargetOpcode::G_LROUND:
981 case TargetOpcode::G_LLROUND: {
982 // Source is always floating point and destination is always integer.
983 OpRegBankIdx = {PMI_FirstGPR, PMI_FirstFPR};
984 break;
985 }
986 }
987
988 // Finally construct the computed mapping.
989 SmallVector<const ValueMapping *, 8> OpdsMapping(NumOperands);
990 for (unsigned Idx = 0; Idx < NumOperands; ++Idx) {
991 if (MI.getOperand(Idx).isReg() && MI.getOperand(Idx).getReg()) {
992 auto Mapping = getValueMapping(OpRegBankIdx[Idx], OpSize[Idx]);
993 if (!Mapping->isValid())
994 return getInvalidInstructionMapping();
995
996 OpdsMapping[Idx] = Mapping;
997 }
998 }
999
1000 return getInstructionMapping(DefaultMappingID, Cost,
1001 getOperandsMapping(OpdsMapping), NumOperands);
1002 }
1003