1 //===-- AMDGPU.h - MachineFunction passes hw codegen --------------*- C++ -*-=//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 /// \file
8 //===----------------------------------------------------------------------===//
9 
10 #ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPU_H
11 #define LLVM_LIB_TARGET_AMDGPU_AMDGPU_H
12 
13 #include "llvm/IR/PassManager.h"
14 #include "llvm/Support/CodeGen.h"
15 
16 namespace llvm {
17 
18 class FunctionPass;
19 class GCNTargetMachine;
20 class ImmutablePass;
21 class MachineFunctionPass;
22 class ModulePass;
23 class Pass;
24 class Target;
25 class TargetMachine;
26 class TargetOptions;
27 class PassRegistry;
28 class Module;
29 
30 // GlobalISel passes
31 void initializeAMDGPUPreLegalizerCombinerPass(PassRegistry &);
32 FunctionPass *createAMDGPUPreLegalizeCombiner(bool IsOptNone);
33 void initializeAMDGPUPostLegalizerCombinerPass(PassRegistry &);
34 FunctionPass *createAMDGPUPostLegalizeCombiner(bool IsOptNone);
35 FunctionPass *createAMDGPURegBankCombiner(bool IsOptNone);
36 void initializeAMDGPURegBankCombinerPass(PassRegistry &);
37 
38 // R600 Passes
39 FunctionPass *createR600VectorRegMerger();
40 FunctionPass *createR600ExpandSpecialInstrsPass();
41 FunctionPass *createR600EmitClauseMarkers();
42 FunctionPass *createR600ClauseMergePass();
43 FunctionPass *createR600Packetizer();
44 FunctionPass *createR600ControlFlowFinalizer();
45 FunctionPass *createAMDGPUCFGStructurizerPass();
46 FunctionPass *createR600ISelDag(TargetMachine *TM, CodeGenOpt::Level OptLevel);
47 
48 // SI Passes
49 FunctionPass *createGCNDPPCombinePass();
50 FunctionPass *createSIAnnotateControlFlowPass();
51 FunctionPass *createSIFoldOperandsPass();
52 FunctionPass *createSIPeepholeSDWAPass();
53 FunctionPass *createSILowerI1CopiesPass();
54 FunctionPass *createSIShrinkInstructionsPass();
55 FunctionPass *createSILoadStoreOptimizerPass();
56 FunctionPass *createSIWholeQuadModePass();
57 FunctionPass *createSIFixControlFlowLiveIntervalsPass();
58 FunctionPass *createSIOptimizeExecMaskingPreRAPass();
59 FunctionPass *createSIOptimizeVGPRLiveRangePass();
60 FunctionPass *createSIFixSGPRCopiesPass();
61 FunctionPass *createSIMemoryLegalizerPass();
62 FunctionPass *createSIInsertWaitcntsPass();
63 FunctionPass *createSIPreAllocateWWMRegsPass();
64 FunctionPass *createSIFormMemoryClausesPass();
65 
66 FunctionPass *createSIPostRABundlerPass();
67 FunctionPass *createAMDGPUSimplifyLibCallsPass(const TargetMachine *);
68 FunctionPass *createAMDGPUUseNativeCallsPass();
69 FunctionPass *createAMDGPUCodeGenPreparePass();
70 FunctionPass *createAMDGPULateCodeGenPreparePass();
71 FunctionPass *createAMDGPUMachineCFGStructurizerPass();
72 FunctionPass *createAMDGPUPropagateAttributesEarlyPass(const TargetMachine *);
73 ModulePass *createAMDGPUPropagateAttributesLatePass(const TargetMachine *);
74 FunctionPass *createAMDGPURewriteOutArgumentsPass();
75 ModulePass *createAMDGPUReplaceLDSUseWithPointerPass();
76 ModulePass *createAMDGPULowerModuleLDSPass();
77 FunctionPass *createSIModeRegisterPass();
78 FunctionPass *createGCNPreRAOptimizationsPass();
79 
80 struct AMDGPUSimplifyLibCallsPass : PassInfoMixin<AMDGPUSimplifyLibCallsPass> {
81   AMDGPUSimplifyLibCallsPass(TargetMachine &TM) : TM(TM) {}
82   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
83 
84 private:
85   TargetMachine &TM;
86 };
87 
88 struct AMDGPUUseNativeCallsPass : PassInfoMixin<AMDGPUUseNativeCallsPass> {
89   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
90 };
91 
92 void initializeAMDGPUDAGToDAGISelPass(PassRegistry&);
93 
94 void initializeAMDGPUMachineCFGStructurizerPass(PassRegistry&);
95 extern char &AMDGPUMachineCFGStructurizerID;
96 
97 void initializeAMDGPUAlwaysInlinePass(PassRegistry&);
98 
99 Pass *createAMDGPUAnnotateKernelFeaturesPass();
100 Pass *createAMDGPUAttributorPass();
101 void initializeAMDGPUAttributorPass(PassRegistry &);
102 void initializeAMDGPUAnnotateKernelFeaturesPass(PassRegistry &);
103 extern char &AMDGPUAnnotateKernelFeaturesID;
104 
105 FunctionPass *createAMDGPUAtomicOptimizerPass();
106 void initializeAMDGPUAtomicOptimizerPass(PassRegistry &);
107 extern char &AMDGPUAtomicOptimizerID;
108 
109 ModulePass *createAMDGPULowerIntrinsicsPass();
110 void initializeAMDGPULowerIntrinsicsPass(PassRegistry &);
111 extern char &AMDGPULowerIntrinsicsID;
112 
113 ModulePass *createAMDGPUFixFunctionBitcastsPass();
114 void initializeAMDGPUFixFunctionBitcastsPass(PassRegistry &);
115 extern char &AMDGPUFixFunctionBitcastsID;
116 
117 ModulePass *createAMDGPUCtorDtorLoweringPass();
118 void initializeAMDGPUCtorDtorLoweringPass(PassRegistry &);
119 extern char &AMDGPUCtorDtorLoweringID;
120 
121 FunctionPass *createAMDGPULowerKernelArgumentsPass();
122 void initializeAMDGPULowerKernelArgumentsPass(PassRegistry &);
123 extern char &AMDGPULowerKernelArgumentsID;
124 
125 ModulePass *createAMDGPULowerKernelAttributesPass();
126 void initializeAMDGPULowerKernelAttributesPass(PassRegistry &);
127 extern char &AMDGPULowerKernelAttributesID;
128 
129 struct AMDGPULowerKernelAttributesPass
130     : PassInfoMixin<AMDGPULowerKernelAttributesPass> {
131   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
132 };
133 
134 void initializeAMDGPUPropagateAttributesEarlyPass(PassRegistry &);
135 extern char &AMDGPUPropagateAttributesEarlyID;
136 
137 struct AMDGPUPropagateAttributesEarlyPass
138     : PassInfoMixin<AMDGPUPropagateAttributesEarlyPass> {
139   AMDGPUPropagateAttributesEarlyPass(TargetMachine &TM) : TM(TM) {}
140   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
141 
142 private:
143   TargetMachine &TM;
144 };
145 
146 void initializeAMDGPUPropagateAttributesLatePass(PassRegistry &);
147 extern char &AMDGPUPropagateAttributesLateID;
148 
149 struct AMDGPUPropagateAttributesLatePass
150     : PassInfoMixin<AMDGPUPropagateAttributesLatePass> {
151   AMDGPUPropagateAttributesLatePass(TargetMachine &TM) : TM(TM) {}
152   PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
153 
154 private:
155   TargetMachine &TM;
156 };
157 
158 void initializeAMDGPUReplaceLDSUseWithPointerPass(PassRegistry &);
159 extern char &AMDGPUReplaceLDSUseWithPointerID;
160 
161 struct AMDGPUReplaceLDSUseWithPointerPass
162     : PassInfoMixin<AMDGPUReplaceLDSUseWithPointerPass> {
163   PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
164 };
165 
166 void initializeAMDGPULowerModuleLDSPass(PassRegistry &);
167 extern char &AMDGPULowerModuleLDSID;
168 
169 struct AMDGPULowerModuleLDSPass : PassInfoMixin<AMDGPULowerModuleLDSPass> {
170   PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
171 };
172 
173 void initializeAMDGPURewriteOutArgumentsPass(PassRegistry &);
174 extern char &AMDGPURewriteOutArgumentsID;
175 
176 void initializeGCNDPPCombinePass(PassRegistry &);
177 extern char &GCNDPPCombineID;
178 
179 void initializeR600ClauseMergePassPass(PassRegistry &);
180 extern char &R600ClauseMergePassID;
181 
182 void initializeR600ControlFlowFinalizerPass(PassRegistry &);
183 extern char &R600ControlFlowFinalizerID;
184 
185 void initializeR600ExpandSpecialInstrsPassPass(PassRegistry &);
186 extern char &R600ExpandSpecialInstrsPassID;
187 
188 void initializeR600VectorRegMergerPass(PassRegistry &);
189 extern char &R600VectorRegMergerID;
190 
191 void initializeR600PacketizerPass(PassRegistry &);
192 extern char &R600PacketizerID;
193 
194 void initializeSIFoldOperandsPass(PassRegistry &);
195 extern char &SIFoldOperandsID;
196 
197 void initializeSIPeepholeSDWAPass(PassRegistry &);
198 extern char &SIPeepholeSDWAID;
199 
200 void initializeSIShrinkInstructionsPass(PassRegistry&);
201 extern char &SIShrinkInstructionsID;
202 
203 void initializeSIFixSGPRCopiesPass(PassRegistry &);
204 extern char &SIFixSGPRCopiesID;
205 
206 void initializeSIFixVGPRCopiesPass(PassRegistry &);
207 extern char &SIFixVGPRCopiesID;
208 
209 void initializeSILowerI1CopiesPass(PassRegistry &);
210 extern char &SILowerI1CopiesID;
211 
212 void initializeSILowerSGPRSpillsPass(PassRegistry &);
213 extern char &SILowerSGPRSpillsID;
214 
215 void initializeSILoadStoreOptimizerPass(PassRegistry &);
216 extern char &SILoadStoreOptimizerID;
217 
218 void initializeSIWholeQuadModePass(PassRegistry &);
219 extern char &SIWholeQuadModeID;
220 
221 void initializeSILowerControlFlowPass(PassRegistry &);
222 extern char &SILowerControlFlowID;
223 
224 void initializeSIPreEmitPeepholePass(PassRegistry &);
225 extern char &SIPreEmitPeepholeID;
226 
227 void initializeSILateBranchLoweringPass(PassRegistry &);
228 extern char &SILateBranchLoweringPassID;
229 
230 void initializeSIOptimizeExecMaskingPass(PassRegistry &);
231 extern char &SIOptimizeExecMaskingID;
232 
233 void initializeSIPreAllocateWWMRegsPass(PassRegistry &);
234 extern char &SIPreAllocateWWMRegsID;
235 
236 void initializeAMDGPUSimplifyLibCallsPass(PassRegistry &);
237 extern char &AMDGPUSimplifyLibCallsID;
238 
239 void initializeAMDGPUUseNativeCallsPass(PassRegistry &);
240 extern char &AMDGPUUseNativeCallsID;
241 
242 void initializeAMDGPUPerfHintAnalysisPass(PassRegistry &);
243 extern char &AMDGPUPerfHintAnalysisID;
244 
245 // Passes common to R600 and SI
246 FunctionPass *createAMDGPUPromoteAlloca();
247 void initializeAMDGPUPromoteAllocaPass(PassRegistry&);
248 extern char &AMDGPUPromoteAllocaID;
249 
250 FunctionPass *createAMDGPUPromoteAllocaToVector();
251 void initializeAMDGPUPromoteAllocaToVectorPass(PassRegistry&);
252 extern char &AMDGPUPromoteAllocaToVectorID;
253 
254 struct AMDGPUPromoteAllocaPass : PassInfoMixin<AMDGPUPromoteAllocaPass> {
255   AMDGPUPromoteAllocaPass(TargetMachine &TM) : TM(TM) {}
256   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
257 
258 private:
259   TargetMachine &TM;
260 };
261 
262 struct AMDGPUPromoteAllocaToVectorPass
263     : PassInfoMixin<AMDGPUPromoteAllocaToVectorPass> {
264   AMDGPUPromoteAllocaToVectorPass(TargetMachine &TM) : TM(TM) {}
265   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
266 
267 private:
268   TargetMachine &TM;
269 };
270 
271 Pass *createAMDGPUStructurizeCFGPass();
272 FunctionPass *createAMDGPUISelDag(
273   TargetMachine *TM = nullptr,
274   CodeGenOpt::Level OptLevel = CodeGenOpt::Default);
275 ModulePass *createAMDGPUAlwaysInlinePass(bool GlobalOpt = true);
276 
277 struct AMDGPUAlwaysInlinePass : PassInfoMixin<AMDGPUAlwaysInlinePass> {
278   AMDGPUAlwaysInlinePass(bool GlobalOpt = true) : GlobalOpt(GlobalOpt) {}
279   PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
280 
281 private:
282   bool GlobalOpt;
283 };
284 
285 ModulePass *createR600OpenCLImageTypeLoweringPass();
286 FunctionPass *createAMDGPUAnnotateUniformValues();
287 
288 ModulePass *createAMDGPUPrintfRuntimeBinding();
289 void initializeAMDGPUPrintfRuntimeBindingPass(PassRegistry&);
290 extern char &AMDGPUPrintfRuntimeBindingID;
291 
292 void initializeAMDGPUResourceUsageAnalysisPass(PassRegistry &);
293 extern char &AMDGPUResourceUsageAnalysisID;
294 
295 struct AMDGPUPrintfRuntimeBindingPass
296     : PassInfoMixin<AMDGPUPrintfRuntimeBindingPass> {
297   PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
298 };
299 
300 ModulePass* createAMDGPUUnifyMetadataPass();
301 void initializeAMDGPUUnifyMetadataPass(PassRegistry&);
302 extern char &AMDGPUUnifyMetadataID;
303 
304 struct AMDGPUUnifyMetadataPass : PassInfoMixin<AMDGPUUnifyMetadataPass> {
305   PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
306 };
307 
308 void initializeSIOptimizeExecMaskingPreRAPass(PassRegistry&);
309 extern char &SIOptimizeExecMaskingPreRAID;
310 
311 void initializeSIOptimizeVGPRLiveRangePass(PassRegistry &);
312 extern char &SIOptimizeVGPRLiveRangeID;
313 
314 void initializeAMDGPUAnnotateUniformValuesPass(PassRegistry&);
315 extern char &AMDGPUAnnotateUniformValuesPassID;
316 
317 void initializeAMDGPUCodeGenPreparePass(PassRegistry&);
318 extern char &AMDGPUCodeGenPrepareID;
319 
320 void initializeAMDGPULateCodeGenPreparePass(PassRegistry &);
321 extern char &AMDGPULateCodeGenPrepareID;
322 
323 void initializeSIAnnotateControlFlowPass(PassRegistry&);
324 extern char &SIAnnotateControlFlowPassID;
325 
326 void initializeSIMemoryLegalizerPass(PassRegistry&);
327 extern char &SIMemoryLegalizerID;
328 
329 void initializeSIModeRegisterPass(PassRegistry&);
330 extern char &SIModeRegisterID;
331 
332 void initializeSIInsertHardClausesPass(PassRegistry &);
333 extern char &SIInsertHardClausesID;
334 
335 void initializeSIInsertWaitcntsPass(PassRegistry&);
336 extern char &SIInsertWaitcntsID;
337 
338 void initializeSIFormMemoryClausesPass(PassRegistry&);
339 extern char &SIFormMemoryClausesID;
340 
341 void initializeSIPostRABundlerPass(PassRegistry&);
342 extern char &SIPostRABundlerID;
343 
344 void initializeAMDGPUUnifyDivergentExitNodesPass(PassRegistry&);
345 extern char &AMDGPUUnifyDivergentExitNodesID;
346 
347 ImmutablePass *createAMDGPUAAWrapperPass();
348 void initializeAMDGPUAAWrapperPassPass(PassRegistry&);
349 ImmutablePass *createAMDGPUExternalAAWrapperPass();
350 void initializeAMDGPUExternalAAWrapperPass(PassRegistry&);
351 
352 void initializeAMDGPUArgumentUsageInfoPass(PassRegistry &);
353 
354 ModulePass *createAMDGPUOpenCLEnqueuedBlockLoweringPass();
355 void initializeAMDGPUOpenCLEnqueuedBlockLoweringPass(PassRegistry &);
356 extern char &AMDGPUOpenCLEnqueuedBlockLoweringID;
357 
358 void initializeGCNNSAReassignPass(PassRegistry &);
359 extern char &GCNNSAReassignID;
360 
361 void initializeGCNPreRAOptimizationsPass(PassRegistry &);
362 extern char &GCNPreRAOptimizationsID;
363 
364 namespace AMDGPU {
365 enum TargetIndex {
366   TI_CONSTDATA_START,
367   TI_SCRATCH_RSRC_DWORD0,
368   TI_SCRATCH_RSRC_DWORD1,
369   TI_SCRATCH_RSRC_DWORD2,
370   TI_SCRATCH_RSRC_DWORD3
371 };
372 }
373 
374 /// OpenCL uses address spaces to differentiate between
375 /// various memory regions on the hardware. On the CPU
376 /// all of the address spaces point to the same memory,
377 /// however on the GPU, each address space points to
378 /// a separate piece of memory that is unique from other
379 /// memory locations.
380 namespace AMDGPUAS {
381   enum : unsigned {
382     // The maximum value for flat, generic, local, private, constant and region.
383     MAX_AMDGPU_ADDRESS = 7,
384 
385     FLAT_ADDRESS = 0,     ///< Address space for flat memory.
386     GLOBAL_ADDRESS = 1,   ///< Address space for global memory (RAT0, VTX0).
387     REGION_ADDRESS = 2,   ///< Address space for region memory. (GDS)
388 
389     CONSTANT_ADDRESS = 4, ///< Address space for constant memory (VTX2).
390     LOCAL_ADDRESS = 3,    ///< Address space for local memory.
391     PRIVATE_ADDRESS = 5,  ///< Address space for private memory.
392 
393     CONSTANT_ADDRESS_32BIT = 6, ///< Address space for 32-bit constant memory.
394 
395     BUFFER_FAT_POINTER = 7, ///< Address space for 160-bit buffer fat pointers.
396 
397     /// Address space for direct addressible parameter memory (CONST0).
398     PARAM_D_ADDRESS = 6,
399     /// Address space for indirect addressible parameter memory (VTX1).
400     PARAM_I_ADDRESS = 7,
401 
402     // Do not re-order the CONSTANT_BUFFER_* enums.  Several places depend on
403     // this order to be able to dynamically index a constant buffer, for
404     // example:
405     //
406     // ConstantBufferAS = CONSTANT_BUFFER_0 + CBIdx
407 
408     CONSTANT_BUFFER_0 = 8,
409     CONSTANT_BUFFER_1 = 9,
410     CONSTANT_BUFFER_2 = 10,
411     CONSTANT_BUFFER_3 = 11,
412     CONSTANT_BUFFER_4 = 12,
413     CONSTANT_BUFFER_5 = 13,
414     CONSTANT_BUFFER_6 = 14,
415     CONSTANT_BUFFER_7 = 15,
416     CONSTANT_BUFFER_8 = 16,
417     CONSTANT_BUFFER_9 = 17,
418     CONSTANT_BUFFER_10 = 18,
419     CONSTANT_BUFFER_11 = 19,
420     CONSTANT_BUFFER_12 = 20,
421     CONSTANT_BUFFER_13 = 21,
422     CONSTANT_BUFFER_14 = 22,
423     CONSTANT_BUFFER_15 = 23,
424 
425     // Some places use this if the address space can't be determined.
426     UNKNOWN_ADDRESS_SPACE = ~0u,
427   };
428 }
429 
430 namespace AMDGPU {
431 
432 // FIXME: Missing constant_32bit
433 inline bool isFlatGlobalAddrSpace(unsigned AS) {
434   return AS == AMDGPUAS::GLOBAL_ADDRESS ||
435          AS == AMDGPUAS::FLAT_ADDRESS ||
436          AS == AMDGPUAS::CONSTANT_ADDRESS ||
437          AS > AMDGPUAS::MAX_AMDGPU_ADDRESS;
438 }
439 }
440 
441 } // End namespace llvm
442 
443 #endif
444