1# RUN: llc -march=amdgcn -mcpu=gfx803 -run-pass si-insert-waitcnts  %s -o - | FileCheck -check-prefixes=CHECK,GFX89 %s
2# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass si-insert-waitcnts  %s -o - | FileCheck -check-prefixes=CHECK,GFX89 %s
3
4--- |
5  define amdgpu_kernel void @flat_zero_waitcnt(i32 addrspace(1)* %global4,
6                                 <4 x i32> addrspace(1)* %global16,
7                                 i32* %flat4,
8                                 <4 x i32>* %flat16) {
9    ret void
10  }
11
12  define amdgpu_kernel void @single_fallthrough_successor_no_end_block_wait() {
13    ret void
14  }
15
16  define amdgpu_kernel void @single_branch_successor_not_next_block() {
17    ret void
18  }
19
20  define amdgpu_kernel void @preexisting_waitcnt() {
21    ret void
22  }
23
24  define amdgpu_kernel void @bundle_no_waitcnt() {
25    ret void
26  }
27
28  define amdgpu_kernel void @preexisting_waitcnt_in_bundle() {
29    ret void
30  }
31
32  define amdgpu_kernel void @insert_in_bundle() {
33    ret void
34  }
35
36  define amdgpu_kernel void @exit_bundle() {
37    ret void
38  }
39
40  define amdgpu_kernel void @cross_bundle() {
41    ret void
42  }
43
44  define amdgpu_kernel void @subregs16bit() {
45    ret void
46  }
47...
48---
49
50# CHECK-LABEL: name: flat_zero_waitcnt
51
52# CHECK-LABEL: bb.0:
53# CHECK: FLAT_LOAD_DWORD
54# CHECK: FLAT_LOAD_DWORDX4
55# Global loads will return in order so we should:
56# s_waitcnt vmcnt(1) lgkmcnt(1)
57# CHECK-NEXT: S_WAITCNT 369
58
59# CHECK-LABEL: bb.1:
60# CHECK: FLAT_LOAD_DWORD
61# GFX89: S_WAITCNT 112
62# CHECK: FLAT_LOAD_DWORDX4
63
64# CHECK-LABEL: bb.2:
65# CHECK: FLAT_LOAD_DWORD
66# GFX89: S_WAITCNT 112
67# CHECK: FLAT_LOAD_DWORDX4
68
69name: flat_zero_waitcnt
70
71body: |
72  bb.0:
73    successors: %bb.1
74    $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %ir.global4)
75    $vgpr3_vgpr4_vgpr5_vgpr6 = FLAT_LOAD_DWORDX4 $vgpr7_vgpr8, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 16 from %ir.global16)
76    $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec
77    S_BRANCH %bb.1
78
79  bb.1:
80    successors: %bb.2
81    $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
82    $vgpr3_vgpr4_vgpr5_vgpr6 = FLAT_LOAD_DWORDX4 $vgpr7_vgpr8, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 16 from %ir.global16)
83    $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec
84    S_BRANCH %bb.2
85
86  bb.2:
87    $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %ir.flat4)
88    $vgpr3_vgpr4_vgpr5_vgpr6 = FLAT_LOAD_DWORDX4 $vgpr7_vgpr8, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 16 from %ir.flat16)
89    $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec
90    S_ENDPGM 0
91...
92---
93# There is only a single fallthrough successor block, so there's no
94# need to wait immediately.
95
96# CHECK-LABEL: name: single_fallthrough_successor_no_end_block_wait
97# CHECK:   $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2
98# CHECK-NOT: S_WAITCNT
99
100# CHECK: bb.1:
101# CHECK-NEXT: V_LSHLREV_B64
102# CHECK-NEXT: S_WAITCNT 112
103# CHECK-NEXT: FLAT_STORE_DWORD
104name: single_fallthrough_successor_no_end_block_wait
105
106body: |
107  bb.0:
108    successors: %bb.1
109    $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
110
111  bb.1:
112    $vgpr3_vgpr4 = V_LSHLREV_B64 4, $vgpr7_vgpr8, implicit $exec
113    FLAT_STORE_DWORD $vgpr3_vgpr4, $vgpr0, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
114    S_ENDPGM 0
115...
116---
117# The block has a single predecessor with a single successor, but it
118# is not the next block so it's non-obvious that the wait is not needed.
119
120
121# CHECK-LABEL: name: single_branch_successor_not_next_block
122
123# CHECK: bb.1
124# CHECK-NEXT: FLAT_STORE_DWORD
125# CHECK-NEXT: S_ENDPGM 0
126
127# CHECK: bb.2:
128# CHECK-NEXT: V_LSHLREV_B64
129# CHECK-NEXT: S_WAITCNT 112
130# CHECK-NEXT: FLAT_STORE_DWORD
131name: single_branch_successor_not_next_block
132
133body: |
134  bb.0:
135    successors: %bb.2
136    $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
137   S_BRANCH %bb.2
138
139  bb.1:
140    FLAT_STORE_DWORD $vgpr8_vgpr9, $vgpr10, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
141    S_ENDPGM 0
142
143  bb.2:
144     $vgpr3_vgpr4 = V_LSHLREV_B64 4, $vgpr7_vgpr8, implicit $exec
145    FLAT_STORE_DWORD $vgpr3_vgpr4, $vgpr0, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
146    S_ENDPGM 0
147...
148
149# CHECK-LABEL: name: preexisting_waitcnt{{$}}
150# CHECK: FLAT_LOAD_DWORD
151# CHECK-NEXT: S_WAITCNT 0
152# CHECK-NOT: S_WAITCNT
153name: preexisting_waitcnt
154tracksRegLiveness: true
155machineFunctionInfo:
156  isEntryFunction: true
157body: |
158  bb.0:
159    liveins: $vgpr1_vgpr2
160    $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
161    S_WAITCNT 0
162    FLAT_STORE_DWORD $vgpr1_vgpr2, $vgpr0, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
163
164...
165
166---
167
168# CHECK-LABEL: name: bundle_no_waitcnt{{$}}
169# CHECK: FLAT_LOAD_DWORD
170# CHECK-NEXT: BUNDLE
171# CHECK-NEXT: S_NOP
172# CHECK-NEXT: S_NOP
173# CHECK-NEXT: }
174# CHECK-NEXT: S_WAITCNT 112
175name: bundle_no_waitcnt
176tracksRegLiveness: true
177machineFunctionInfo:
178  isEntryFunction: true
179body: |
180  bb.0:
181    liveins: $vgpr1_vgpr2
182    $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
183    BUNDLE {
184      S_NOP 0
185      S_NOP 0
186    }
187    FLAT_STORE_DWORD $vgpr1_vgpr2, $vgpr0, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
188
189...
190
191---
192
193# See the waitcnt inside the bundle and don't insert an extra
194# CHECK-LABEL: name: preexisting_waitcnt_in_bundle{{$}}
195# CHECK: FLAT_LOAD_DWORD
196# CHECK: S_WAITCNT 0
197# CHECK-NOT: S_WAITCNT
198name: preexisting_waitcnt_in_bundle
199tracksRegLiveness: true
200machineFunctionInfo:
201  isEntryFunction: true
202body: |
203  bb.0:
204    liveins: $vgpr1_vgpr2
205    $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
206    BUNDLE {
207      S_NOP 0
208      S_WAITCNT 0
209    }
210    FLAT_STORE_DWORD $vgpr1_vgpr2, $vgpr0, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
211
212...
213
214---
215
216# Def and use inside bundle
217# CHECK-LABEL: name: insert_in_bundle{{$}}
218# CHECK: BUNDLE implicit-def $vgpr0, implicit $vgpr1_vgpr2 {
219# CHECK-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
220# CHECK-NEXT: S_WAITCNT 112
221# CHECK-NEXT: FLAT_STORE_DWORD $vgpr1_vgpr2, internal $vgpr0, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
222# CHECK-NEXT: }
223
224name: insert_in_bundle
225tracksRegLiveness: true
226machineFunctionInfo:
227  isEntryFunction: true
228body: |
229  bb.0:
230    liveins: $vgpr1_vgpr2
231    BUNDLE implicit-def $vgpr0, implicit $vgpr1_vgpr2 {
232    $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
233    FLAT_STORE_DWORD $vgpr1_vgpr2, internal $vgpr0, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
234    }
235...
236
237---
238
239# Def is last instruction in bundle, use is outside bundle
240
241# CHECK-LABEL: name: exit_bundle{{$}}
242# CHECK: BUNDLE implicit-def $vgpr0, implicit $vgpr1_vgpr2 {
243# CHECK-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
244# CHECK-NEXT: }
245# CHECK-NEXT: S_WAITCNT 112
246# CHECK-NEXT: FLAT_STORE_DWORD $vgpr1_vgpr2, $vgpr0, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
247
248name: exit_bundle
249tracksRegLiveness: true
250machineFunctionInfo:
251  isEntryFunction: true
252body: |
253  bb.0:
254    liveins: $vgpr1_vgpr2
255    BUNDLE implicit-def $vgpr0, implicit $vgpr1_vgpr2 {
256    $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
257    }
258
259    FLAT_STORE_DWORD $vgpr1_vgpr2, $vgpr0, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
260
261...
262
263---
264
265# Def is in bundle, use is in another bundle
266
267# CHECK-LABEL: name: cross_bundle{{$}}
268# CHECK: BUNDLE implicit-def $vgpr0, implicit $vgpr1_vgpr2 {
269# CHECK-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
270# CHECK-NEXT: }
271# CHECK-NEXT: S_WAITCNT 112
272# CHECK-NEXT: BUNDLE implicit $vgpr0, implicit $vgpr1_vgpr2 {
273# CHECK-NEXT: FLAT_STORE_DWORD $vgpr1_vgpr2, $vgpr0, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
274# CHECK-NEXT: }
275
276name: cross_bundle
277tracksRegLiveness: true
278machineFunctionInfo:
279  isEntryFunction: true
280body: |
281  bb.0:
282    liveins: $vgpr1_vgpr2
283    BUNDLE implicit-def $vgpr0, implicit $vgpr1_vgpr2 {
284    $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
285    }
286    BUNDLE implicit $vgpr0, implicit $vgpr1_vgpr2 {
287      FLAT_STORE_DWORD $vgpr1_vgpr2, $vgpr0, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
288    }
289...
290
291---
292# CHECK-LABEL: name: subregs16bit
293# CHECK: S_WAITCNT 112
294# CHECK-NEXT: V_NOP_e32
295
296name: subregs16bit
297machineFunctionInfo:
298  isEntryFunction: true
299body: |
300  bb.0:
301    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4
302      $vgpr0 = FLAT_LOAD_USHORT killed $vgpr0_vgpr1, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
303      $vgpr1 = FLAT_LOAD_USHORT killed $vgpr2_vgpr3, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
304      V_NOP_e32 implicit $exec, implicit $vgpr0_lo16, implicit $vgpr1_lo16
305...
306