1 //===-- AMDGPUSubtarget.cpp - AMDGPU Subtarget Information ----------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file
11 /// \brief Implements the AMDGPU specific subclass of TargetSubtarget.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "AMDGPUSubtarget.h"
16 #include "AMDGPU.h"
17 #include "AMDGPUTargetMachine.h"
18 #include "AMDGPUCallLowering.h"
19 #include "AMDGPUInstructionSelector.h"
20 #include "AMDGPULegalizerInfo.h"
21 #include "AMDGPURegisterBankInfo.h"
22 #include "SIMachineFunctionInfo.h"
23 #include "llvm/ADT/SmallString.h"
24 #include "llvm/CodeGen/MachineScheduler.h"
25 #include "llvm/IR/MDBuilder.h"
26 #include "llvm/Target/TargetFrameLowering.h"
27 #include <algorithm>
28 
29 using namespace llvm;
30 
31 #define DEBUG_TYPE "amdgpu-subtarget"
32 
33 #define GET_SUBTARGETINFO_TARGET_DESC
34 #define GET_SUBTARGETINFO_CTOR
35 #include "AMDGPUGenSubtargetInfo.inc"
36 
37 AMDGPUSubtarget::~AMDGPUSubtarget() = default;
38 
39 AMDGPUSubtarget &
40 AMDGPUSubtarget::initializeSubtargetDependencies(const Triple &TT,
41                                                  StringRef GPU, StringRef FS) {
42   // Determine default and user-specified characteristics
43   // On SI+, we want FP64 denormals to be on by default. FP32 denormals can be
44   // enabled, but some instructions do not respect them and they run at the
45   // double precision rate, so don't enable by default.
46   //
47   // We want to be able to turn these off, but making this a subtarget feature
48   // for SI has the unhelpful behavior that it unsets everything else if you
49   // disable it.
50 
51   SmallString<256> FullFS("+promote-alloca,+fp64-fp16-denormals,+dx10-clamp,+load-store-opt,");
52   if (isAmdHsaOS()) // Turn on FlatForGlobal for HSA.
53     FullFS += "+flat-address-space,+flat-for-global,+unaligned-buffer-access,+trap-handler,";
54 
55   FullFS += FS;
56 
57   ParseSubtargetFeatures(GPU, FullFS);
58 
59   // Unless +-flat-for-global is specified, turn on FlatForGlobal for all OS-es
60   // on VI and newer hardware to avoid assertion failures due to missing ADDR64
61   // variants of MUBUF instructions.
62   if (!hasAddr64() && !FS.contains("flat-for-global")) {
63     FlatForGlobal = true;
64   }
65 
66   // FIXME: I don't think think Evergreen has any useful support for
67   // denormals, but should be checked. Should we issue a warning somewhere
68   // if someone tries to enable these?
69   if (getGeneration() <= AMDGPUSubtarget::NORTHERN_ISLANDS) {
70     FP64FP16Denormals = false;
71     FP32Denormals = false;
72   }
73 
74   // Set defaults if needed.
75   if (MaxPrivateElementSize == 0)
76     MaxPrivateElementSize = 4;
77 
78   if (LDSBankCount == 0)
79     LDSBankCount = 32;
80 
81   if (TT.getArch() == Triple::amdgcn) {
82     if (LocalMemorySize == 0)
83       LocalMemorySize = 32768;
84 
85     // Do something sensible for unspecified target.
86     if (!HasMovrel && !HasVGPRIndexMode)
87       HasMovrel = true;
88   }
89 
90   return *this;
91 }
92 
93 AMDGPUSubtarget::AMDGPUSubtarget(const Triple &TT, StringRef GPU, StringRef FS,
94                                  const TargetMachine &TM)
95   : AMDGPUGenSubtargetInfo(TT, GPU, FS),
96     TargetTriple(TT),
97     Gen(TT.getArch() == Triple::amdgcn ? SOUTHERN_ISLANDS : R600),
98     IsaVersion(ISAVersion0_0_0),
99     WavefrontSize(0),
100     LocalMemorySize(0),
101     LDSBankCount(0),
102     MaxPrivateElementSize(0),
103 
104     FastFMAF32(false),
105     HalfRate64Ops(false),
106 
107     FP32Denormals(false),
108     FP64FP16Denormals(false),
109     FPExceptions(false),
110     DX10Clamp(false),
111     FlatForGlobal(false),
112     AutoWaitcntBeforeBarrier(false),
113     CodeObjectV3(false),
114     UnalignedScratchAccess(false),
115     UnalignedBufferAccess(false),
116 
117     HasApertureRegs(false),
118     EnableXNACK(false),
119     TrapHandler(false),
120     DebuggerInsertNops(false),
121     DebuggerReserveRegs(false),
122     DebuggerEmitPrologue(false),
123 
124     EnableVGPRSpilling(false),
125     EnablePromoteAlloca(false),
126     EnableLoadStoreOpt(false),
127     EnableUnsafeDSOffsetFolding(false),
128     EnableSIScheduler(false),
129     DumpCode(false),
130 
131     FP64(false),
132     IsGCN(false),
133     GCN3Encoding(false),
134     CIInsts(false),
135     GFX9Insts(false),
136     SGPRInitBug(false),
137     HasSMemRealTime(false),
138     Has16BitInsts(false),
139     HasIntClamp(false),
140     HasVOP3PInsts(false),
141     HasMovrel(false),
142     HasVGPRIndexMode(false),
143     HasScalarStores(false),
144     HasInv2PiInlineImm(false),
145     HasSDWA(false),
146     HasSDWAOmod(false),
147     HasSDWAScalar(false),
148     HasSDWASdst(false),
149     HasSDWAMac(false),
150     HasSDWAOutModsVOPC(false),
151     HasDPP(false),
152     FlatAddressSpace(false),
153     FlatInstOffsets(false),
154     FlatGlobalInsts(false),
155     FlatScratchInsts(false),
156     AddNoCarryInsts(false),
157 
158     R600ALUInst(false),
159     CaymanISA(false),
160     CFALUBug(false),
161     HasVertexCache(false),
162     TexVTXClauseSize(0),
163     ScalarizeGlobal(false),
164 
165     FeatureDisable(false),
166     InstrItins(getInstrItineraryForCPU(GPU)) {
167   AS = AMDGPU::getAMDGPUAS(TT);
168   initializeSubtargetDependencies(TT, GPU, FS);
169 }
170 
171 unsigned AMDGPUSubtarget::getMaxLocalMemSizeWithWaveCount(unsigned NWaves,
172   const Function &F) const {
173   if (NWaves == 1)
174     return getLocalMemorySize();
175   unsigned WorkGroupSize = getFlatWorkGroupSizes(F).second;
176   unsigned WorkGroupsPerCu = getMaxWorkGroupsPerCU(WorkGroupSize);
177   unsigned MaxWaves = getMaxWavesPerEU();
178   return getLocalMemorySize() * MaxWaves / WorkGroupsPerCu / NWaves;
179 }
180 
181 unsigned AMDGPUSubtarget::getOccupancyWithLocalMemSize(uint32_t Bytes,
182   const Function &F) const {
183   unsigned WorkGroupSize = getFlatWorkGroupSizes(F).second;
184   unsigned WorkGroupsPerCu = getMaxWorkGroupsPerCU(WorkGroupSize);
185   unsigned MaxWaves = getMaxWavesPerEU();
186   unsigned Limit = getLocalMemorySize() * MaxWaves / WorkGroupsPerCu;
187   unsigned NumWaves = Limit / (Bytes ? Bytes : 1u);
188   NumWaves = std::min(NumWaves, MaxWaves);
189   NumWaves = std::max(NumWaves, 1u);
190   return NumWaves;
191 }
192 
193 std::pair<unsigned, unsigned>
194 AMDGPUSubtarget::getDefaultFlatWorkGroupSize(CallingConv::ID CC) const {
195   switch (CC) {
196   case CallingConv::AMDGPU_CS:
197   case CallingConv::AMDGPU_KERNEL:
198   case CallingConv::SPIR_KERNEL:
199     return std::make_pair(getWavefrontSize() * 2, getWavefrontSize() * 4);
200   case CallingConv::AMDGPU_VS:
201   case CallingConv::AMDGPU_LS:
202   case CallingConv::AMDGPU_HS:
203   case CallingConv::AMDGPU_ES:
204   case CallingConv::AMDGPU_GS:
205   case CallingConv::AMDGPU_PS:
206     return std::make_pair(1, getWavefrontSize());
207   default:
208     return std::make_pair(1, 16 * getWavefrontSize());
209   }
210 }
211 
212 std::pair<unsigned, unsigned> AMDGPUSubtarget::getFlatWorkGroupSizes(
213   const Function &F) const {
214   // FIXME: 1024 if function.
215   // Default minimum/maximum flat work group sizes.
216   std::pair<unsigned, unsigned> Default =
217     getDefaultFlatWorkGroupSize(F.getCallingConv());
218 
219   // TODO: Do not process "amdgpu-max-work-group-size" attribute once mesa
220   // starts using "amdgpu-flat-work-group-size" attribute.
221   Default.second = AMDGPU::getIntegerAttribute(
222     F, "amdgpu-max-work-group-size", Default.second);
223   Default.first = std::min(Default.first, Default.second);
224 
225   // Requested minimum/maximum flat work group sizes.
226   std::pair<unsigned, unsigned> Requested = AMDGPU::getIntegerPairAttribute(
227     F, "amdgpu-flat-work-group-size", Default);
228 
229   // Make sure requested minimum is less than requested maximum.
230   if (Requested.first > Requested.second)
231     return Default;
232 
233   // Make sure requested values do not violate subtarget's specifications.
234   if (Requested.first < getMinFlatWorkGroupSize())
235     return Default;
236   if (Requested.second > getMaxFlatWorkGroupSize())
237     return Default;
238 
239   return Requested;
240 }
241 
242 std::pair<unsigned, unsigned> AMDGPUSubtarget::getWavesPerEU(
243   const Function &F) const {
244   // Default minimum/maximum number of waves per execution unit.
245   std::pair<unsigned, unsigned> Default(1, getMaxWavesPerEU());
246 
247   // Default/requested minimum/maximum flat work group sizes.
248   std::pair<unsigned, unsigned> FlatWorkGroupSizes = getFlatWorkGroupSizes(F);
249 
250   // If minimum/maximum flat work group sizes were explicitly requested using
251   // "amdgpu-flat-work-group-size" attribute, then set default minimum/maximum
252   // number of waves per execution unit to values implied by requested
253   // minimum/maximum flat work group sizes.
254   unsigned MinImpliedByFlatWorkGroupSize =
255     getMaxWavesPerEU(FlatWorkGroupSizes.second);
256   bool RequestedFlatWorkGroupSize = false;
257 
258   // TODO: Do not process "amdgpu-max-work-group-size" attribute once mesa
259   // starts using "amdgpu-flat-work-group-size" attribute.
260   if (F.hasFnAttribute("amdgpu-max-work-group-size") ||
261       F.hasFnAttribute("amdgpu-flat-work-group-size")) {
262     Default.first = MinImpliedByFlatWorkGroupSize;
263     RequestedFlatWorkGroupSize = true;
264   }
265 
266   // Requested minimum/maximum number of waves per execution unit.
267   std::pair<unsigned, unsigned> Requested = AMDGPU::getIntegerPairAttribute(
268     F, "amdgpu-waves-per-eu", Default, true);
269 
270   // Make sure requested minimum is less than requested maximum.
271   if (Requested.second && Requested.first > Requested.second)
272     return Default;
273 
274   // Make sure requested values do not violate subtarget's specifications.
275   if (Requested.first < getMinWavesPerEU() ||
276       Requested.first > getMaxWavesPerEU())
277     return Default;
278   if (Requested.second > getMaxWavesPerEU())
279     return Default;
280 
281   // Make sure requested values are compatible with values implied by requested
282   // minimum/maximum flat work group sizes.
283   if (RequestedFlatWorkGroupSize &&
284       Requested.first < MinImpliedByFlatWorkGroupSize)
285     return Default;
286 
287   return Requested;
288 }
289 
290 bool AMDGPUSubtarget::makeLIDRangeMetadata(Instruction *I) const {
291   Function *Kernel = I->getParent()->getParent();
292   unsigned MinSize = 0;
293   unsigned MaxSize = getFlatWorkGroupSizes(*Kernel).second;
294   bool IdQuery = false;
295 
296   // If reqd_work_group_size is present it narrows value down.
297   if (auto *CI = dyn_cast<CallInst>(I)) {
298     const Function *F = CI->getCalledFunction();
299     if (F) {
300       unsigned Dim = UINT_MAX;
301       switch (F->getIntrinsicID()) {
302       case Intrinsic::amdgcn_workitem_id_x:
303       case Intrinsic::r600_read_tidig_x:
304         IdQuery = true;
305         LLVM_FALLTHROUGH;
306       case Intrinsic::r600_read_local_size_x:
307         Dim = 0;
308         break;
309       case Intrinsic::amdgcn_workitem_id_y:
310       case Intrinsic::r600_read_tidig_y:
311         IdQuery = true;
312         LLVM_FALLTHROUGH;
313       case Intrinsic::r600_read_local_size_y:
314         Dim = 1;
315         break;
316       case Intrinsic::amdgcn_workitem_id_z:
317       case Intrinsic::r600_read_tidig_z:
318         IdQuery = true;
319         LLVM_FALLTHROUGH;
320       case Intrinsic::r600_read_local_size_z:
321         Dim = 2;
322         break;
323       default:
324         break;
325       }
326       if (Dim <= 3) {
327         if (auto Node = Kernel->getMetadata("reqd_work_group_size"))
328           if (Node->getNumOperands() == 3)
329             MinSize = MaxSize = mdconst::extract<ConstantInt>(
330                                   Node->getOperand(Dim))->getZExtValue();
331       }
332     }
333   }
334 
335   if (!MaxSize)
336     return false;
337 
338   // Range metadata is [Lo, Hi). For ID query we need to pass max size
339   // as Hi. For size query we need to pass Hi + 1.
340   if (IdQuery)
341     MinSize = 0;
342   else
343     ++MaxSize;
344 
345   MDBuilder MDB(I->getContext());
346   MDNode *MaxWorkGroupSizeRange = MDB.createRange(APInt(32, MinSize),
347                                                   APInt(32, MaxSize));
348   I->setMetadata(LLVMContext::MD_range, MaxWorkGroupSizeRange);
349   return true;
350 }
351 
352 R600Subtarget::R600Subtarget(const Triple &TT, StringRef GPU, StringRef FS,
353                              const TargetMachine &TM) :
354   AMDGPUSubtarget(TT, GPU, FS, TM),
355   InstrInfo(*this),
356   FrameLowering(TargetFrameLowering::StackGrowsUp, getStackAlignment(), 0),
357   TLInfo(TM, *this) {}
358 
359 SISubtarget::SISubtarget(const Triple &TT, StringRef GPU, StringRef FS,
360                          const TargetMachine &TM)
361     : AMDGPUSubtarget(TT, GPU, FS, TM), InstrInfo(*this),
362       FrameLowering(TargetFrameLowering::StackGrowsUp, getStackAlignment(), 0),
363       TLInfo(TM, *this) {
364   CallLoweringInfo.reset(new AMDGPUCallLowering(*getTargetLowering()));
365   Legalizer.reset(new AMDGPULegalizerInfo());
366 
367   RegBankInfo.reset(new AMDGPURegisterBankInfo(*getRegisterInfo()));
368   InstSelector.reset(new AMDGPUInstructionSelector(
369       *this, *static_cast<AMDGPURegisterBankInfo *>(RegBankInfo.get())));
370 }
371 
372 void SISubtarget::overrideSchedPolicy(MachineSchedPolicy &Policy,
373                                       unsigned NumRegionInstrs) const {
374   // Track register pressure so the scheduler can try to decrease
375   // pressure once register usage is above the threshold defined by
376   // SIRegisterInfo::getRegPressureSetLimit()
377   Policy.ShouldTrackPressure = true;
378 
379   // Enabling both top down and bottom up scheduling seems to give us less
380   // register spills than just using one of these approaches on its own.
381   Policy.OnlyTopDown = false;
382   Policy.OnlyBottomUp = false;
383 
384   // Enabling ShouldTrackLaneMasks crashes the SI Machine Scheduler.
385   if (!enableSIScheduler())
386     Policy.ShouldTrackLaneMasks = true;
387 }
388 
389 bool SISubtarget::isVGPRSpillingEnabled(const Function& F) const {
390   return EnableVGPRSpilling || !AMDGPU::isShader(F.getCallingConv());
391 }
392 
393 unsigned SISubtarget::getKernArgSegmentSize(const MachineFunction &MF,
394                                             unsigned ExplicitArgBytes) const {
395   unsigned ImplicitBytes = getImplicitArgNumBytes(MF);
396   if (ImplicitBytes == 0)
397     return ExplicitArgBytes;
398 
399   unsigned Alignment = getAlignmentForImplicitArgPtr();
400   return alignTo(ExplicitArgBytes, Alignment) + ImplicitBytes;
401 }
402 
403 unsigned SISubtarget::getOccupancyWithNumSGPRs(unsigned SGPRs) const {
404   if (getGeneration() >= SISubtarget::VOLCANIC_ISLANDS) {
405     if (SGPRs <= 80)
406       return 10;
407     if (SGPRs <= 88)
408       return 9;
409     if (SGPRs <= 100)
410       return 8;
411     return 7;
412   }
413   if (SGPRs <= 48)
414     return 10;
415   if (SGPRs <= 56)
416     return 9;
417   if (SGPRs <= 64)
418     return 8;
419   if (SGPRs <= 72)
420     return 7;
421   if (SGPRs <= 80)
422     return 6;
423   return 5;
424 }
425 
426 unsigned SISubtarget::getOccupancyWithNumVGPRs(unsigned VGPRs) const {
427   if (VGPRs <= 24)
428     return 10;
429   if (VGPRs <= 28)
430     return 9;
431   if (VGPRs <= 32)
432     return 8;
433   if (VGPRs <= 36)
434     return 7;
435   if (VGPRs <= 40)
436     return 6;
437   if (VGPRs <= 48)
438     return 5;
439   if (VGPRs <= 64)
440     return 4;
441   if (VGPRs <= 84)
442     return 3;
443   if (VGPRs <= 128)
444     return 2;
445   return 1;
446 }
447 
448 unsigned SISubtarget::getReservedNumSGPRs(const MachineFunction &MF) const {
449   const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
450   if (MFI.hasFlatScratchInit()) {
451     if (getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
452       return 6; // FLAT_SCRATCH, XNACK, VCC (in that order).
453     if (getGeneration() == AMDGPUSubtarget::SEA_ISLANDS)
454       return 4; // FLAT_SCRATCH, VCC (in that order).
455   }
456 
457   if (isXNACKEnabled())
458     return 4; // XNACK, VCC (in that order).
459   return 2; // VCC.
460 }
461 
462 unsigned SISubtarget::getMaxNumSGPRs(const MachineFunction &MF) const {
463   const Function &F = *MF.getFunction();
464   const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
465 
466   // Compute maximum number of SGPRs function can use using default/requested
467   // minimum number of waves per execution unit.
468   std::pair<unsigned, unsigned> WavesPerEU = MFI.getWavesPerEU();
469   unsigned MaxNumSGPRs = getMaxNumSGPRs(WavesPerEU.first, false);
470   unsigned MaxAddressableNumSGPRs = getMaxNumSGPRs(WavesPerEU.first, true);
471 
472   // Check if maximum number of SGPRs was explicitly requested using
473   // "amdgpu-num-sgpr" attribute.
474   if (F.hasFnAttribute("amdgpu-num-sgpr")) {
475     unsigned Requested = AMDGPU::getIntegerAttribute(
476       F, "amdgpu-num-sgpr", MaxNumSGPRs);
477 
478     // Make sure requested value does not violate subtarget's specifications.
479     if (Requested && (Requested <= getReservedNumSGPRs(MF)))
480       Requested = 0;
481 
482     // If more SGPRs are required to support the input user/system SGPRs,
483     // increase to accommodate them.
484     //
485     // FIXME: This really ends up using the requested number of SGPRs + number
486     // of reserved special registers in total. Theoretically you could re-use
487     // the last input registers for these special registers, but this would
488     // require a lot of complexity to deal with the weird aliasing.
489     unsigned InputNumSGPRs = MFI.getNumPreloadedSGPRs();
490     if (Requested && Requested < InputNumSGPRs)
491       Requested = InputNumSGPRs;
492 
493     // Make sure requested value is compatible with values implied by
494     // default/requested minimum/maximum number of waves per execution unit.
495     if (Requested && Requested > getMaxNumSGPRs(WavesPerEU.first, false))
496       Requested = 0;
497     if (WavesPerEU.second &&
498         Requested && Requested < getMinNumSGPRs(WavesPerEU.second))
499       Requested = 0;
500 
501     if (Requested)
502       MaxNumSGPRs = Requested;
503   }
504 
505   if (hasSGPRInitBug())
506     MaxNumSGPRs = AMDGPU::IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG;
507 
508   return std::min(MaxNumSGPRs - getReservedNumSGPRs(MF),
509                   MaxAddressableNumSGPRs);
510 }
511 
512 unsigned SISubtarget::getMaxNumVGPRs(const MachineFunction &MF) const {
513   const Function &F = *MF.getFunction();
514   const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
515 
516   // Compute maximum number of VGPRs function can use using default/requested
517   // minimum number of waves per execution unit.
518   std::pair<unsigned, unsigned> WavesPerEU = MFI.getWavesPerEU();
519   unsigned MaxNumVGPRs = getMaxNumVGPRs(WavesPerEU.first);
520 
521   // Check if maximum number of VGPRs was explicitly requested using
522   // "amdgpu-num-vgpr" attribute.
523   if (F.hasFnAttribute("amdgpu-num-vgpr")) {
524     unsigned Requested = AMDGPU::getIntegerAttribute(
525       F, "amdgpu-num-vgpr", MaxNumVGPRs);
526 
527     // Make sure requested value does not violate subtarget's specifications.
528     if (Requested && Requested <= getReservedNumVGPRs(MF))
529       Requested = 0;
530 
531     // Make sure requested value is compatible with values implied by
532     // default/requested minimum/maximum number of waves per execution unit.
533     if (Requested && Requested > getMaxNumVGPRs(WavesPerEU.first))
534       Requested = 0;
535     if (WavesPerEU.second &&
536         Requested && Requested < getMinNumVGPRs(WavesPerEU.second))
537       Requested = 0;
538 
539     if (Requested)
540       MaxNumVGPRs = Requested;
541   }
542 
543   return MaxNumVGPRs - getReservedNumVGPRs(MF);
544 }
545 
546 struct MemOpClusterMutation : ScheduleDAGMutation {
547   const SIInstrInfo *TII;
548 
549   MemOpClusterMutation(const SIInstrInfo *tii) : TII(tii) {}
550 
551   void apply(ScheduleDAGInstrs *DAGInstrs) override {
552     ScheduleDAGMI *DAG = static_cast<ScheduleDAGMI*>(DAGInstrs);
553 
554     SUnit *SUa = nullptr;
555     // Search for two consequent memory operations and link them
556     // to prevent scheduler from moving them apart.
557     // In DAG pre-process SUnits are in the original order of
558     // the instructions before scheduling.
559     for (SUnit &SU : DAG->SUnits) {
560       MachineInstr &MI2 = *SU.getInstr();
561       if (!MI2.mayLoad() && !MI2.mayStore()) {
562         SUa = nullptr;
563         continue;
564       }
565       if (!SUa) {
566         SUa = &SU;
567         continue;
568       }
569 
570       MachineInstr &MI1 = *SUa->getInstr();
571       if ((TII->isVMEM(MI1) && TII->isVMEM(MI2)) ||
572           (TII->isFLAT(MI1) && TII->isFLAT(MI2)) ||
573           (TII->isSMRD(MI1) && TII->isSMRD(MI2)) ||
574           (TII->isDS(MI1)   && TII->isDS(MI2))) {
575         SU.addPredBarrier(SUa);
576 
577         for (const SDep &SI : SU.Preds) {
578           if (SI.getSUnit() != SUa)
579             SUa->addPred(SDep(SI.getSUnit(), SDep::Artificial));
580         }
581 
582         if (&SU != &DAG->ExitSU) {
583           for (const SDep &SI : SUa->Succs) {
584             if (SI.getSUnit() != &SU)
585               SI.getSUnit()->addPred(SDep(&SU, SDep::Artificial));
586           }
587         }
588       }
589 
590       SUa = &SU;
591     }
592   }
593 };
594 
595 void SISubtarget::getPostRAMutations(
596     std::vector<std::unique_ptr<ScheduleDAGMutation>> &Mutations) const {
597   Mutations.push_back(llvm::make_unique<MemOpClusterMutation>(&InstrInfo));
598 }
599