1# RUN: llc -march=amdgcn -mcpu=gfx803 -run-pass si-insert-waitcnts  %s -o - | FileCheck -check-prefixes=CHECK,GFX89 %s
2# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass si-insert-waitcnts  %s -o - | FileCheck -check-prefixes=CHECK,GFX89 %s
3
4--- |
5  define amdgpu_kernel void @flat_zero_waitcnt(i32 addrspace(1)* %global4,
6                                 <4 x i32> addrspace(1)* %global16,
7                                 i32* %flat4,
8                                 <4 x i32>* %flat16) {
9    ret void
10  }
11
12  define amdgpu_kernel void @single_fallthrough_successor_no_end_block_wait() {
13    ret void
14  }
15
16  define amdgpu_kernel void @single_branch_successor_not_next_block() {
17    ret void
18  }
19
20  define amdgpu_kernel void @preexisting_waitcnt() {
21    ret void
22  }
23
24  define amdgpu_kernel void @bundle_no_waitcnt() {
25    ret void
26  }
27
28  define amdgpu_kernel void @preexisting_waitcnt_in_bundle() {
29    ret void
30  }
31
32  define amdgpu_kernel void @insert_in_bundle() {
33    ret void
34  }
35
36  define amdgpu_kernel void @exit_bundle() {
37    ret void
38  }
39
40  define amdgpu_kernel void @cross_bundle() {
41    ret void
42  }
43
44...
45---
46
47# CHECK-LABEL: name: flat_zero_waitcnt
48
49# CHECK-LABEL: bb.0:
50# CHECK: FLAT_LOAD_DWORD
51# CHECK: FLAT_LOAD_DWORDX4
52# Global loads will return in order so we should:
53# s_waitcnt vmcnt(1) lgkmcnt(1)
54# CHECK-NEXT: S_WAITCNT 369
55
56# CHECK-LABEL: bb.1:
57# CHECK: FLAT_LOAD_DWORD
58# GFX89: S_WAITCNT 112
59# CHECK: FLAT_LOAD_DWORDX4
60
61# CHECK-LABEL: bb.2:
62# CHECK: FLAT_LOAD_DWORD
63# GFX89: S_WAITCNT 112
64# CHECK: FLAT_LOAD_DWORDX4
65
66name: flat_zero_waitcnt
67
68body: |
69  bb.0:
70    successors: %bb.1
71    $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %ir.global4)
72    $vgpr3_vgpr4_vgpr5_vgpr6 = FLAT_LOAD_DWORDX4 $vgpr7_vgpr8, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 16 from %ir.global16)
73    $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec
74    S_BRANCH %bb.1
75
76  bb.1:
77    successors: %bb.2
78    $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
79    $vgpr3_vgpr4_vgpr5_vgpr6 = FLAT_LOAD_DWORDX4 $vgpr7_vgpr8, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 16 from %ir.global16)
80    $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec
81    S_BRANCH %bb.2
82
83  bb.2:
84    $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %ir.flat4)
85    $vgpr3_vgpr4_vgpr5_vgpr6 = FLAT_LOAD_DWORDX4 $vgpr7_vgpr8, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 16 from %ir.flat16)
86    $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec
87    S_ENDPGM 0
88...
89---
90# There is only a single fallthrough successor block, so there's no
91# need to wait immediately.
92
93# CHECK-LABEL: name: single_fallthrough_successor_no_end_block_wait
94# CHECK:   $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2
95# CHECK-NOT: S_WAITCNT
96
97# CHECK: bb.1:
98# CHECK-NEXT: V_LSHLREV_B64
99# CHECK-NEXT: S_WAITCNT 112
100# CHECK-NEXT: FLAT_STORE_DWORD
101name: single_fallthrough_successor_no_end_block_wait
102
103body: |
104  bb.0:
105    successors: %bb.1
106    $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
107
108  bb.1:
109    $vgpr3_vgpr4 = V_LSHLREV_B64 4, $vgpr7_vgpr8, implicit $exec
110    FLAT_STORE_DWORD $vgpr3_vgpr4, $vgpr0, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
111    S_ENDPGM 0
112...
113---
114# The block has a single predecessor with a single successor, but it
115# is not the next block so it's non-obvious that the wait is not needed.
116
117
118# CHECK-LABEL: name: single_branch_successor_not_next_block
119
120# CHECK: bb.1
121# CHECK-NEXT: FLAT_STORE_DWORD
122# CHECK-NEXT: S_ENDPGM 0
123
124# CHECK: bb.2:
125# CHECK-NEXT: V_LSHLREV_B64
126# CHECK-NEXT: S_WAITCNT 112
127# CHECK-NEXT: FLAT_STORE_DWORD
128name: single_branch_successor_not_next_block
129
130body: |
131  bb.0:
132    successors: %bb.2
133    $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
134   S_BRANCH %bb.2
135
136  bb.1:
137    FLAT_STORE_DWORD $vgpr8_vgpr9, $vgpr10, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
138    S_ENDPGM 0
139
140  bb.2:
141     $vgpr3_vgpr4 = V_LSHLREV_B64 4, $vgpr7_vgpr8, implicit $exec
142    FLAT_STORE_DWORD $vgpr3_vgpr4, $vgpr0, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
143    S_ENDPGM 0
144...
145
146# CHECK-LABEL: name: preexisting_waitcnt{{$}}
147# CHECK: FLAT_LOAD_DWORD
148# CHECK-NEXT: S_WAITCNT 0
149# CHECK-NOT: S_WAITCNT
150name: preexisting_waitcnt
151tracksRegLiveness: true
152machineFunctionInfo:
153  isEntryFunction: true
154body: |
155  bb.0:
156    liveins: $vgpr1_vgpr2
157    $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
158    S_WAITCNT 0
159    FLAT_STORE_DWORD $vgpr1_vgpr2, $vgpr0, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
160
161...
162
163---
164
165# CHECK-LABEL: name: bundle_no_waitcnt{{$}}
166# CHECK: FLAT_LOAD_DWORD
167# CHECK-NEXT: BUNDLE
168# CHECK-NEXT: S_NOP
169# CHECK-NEXT: S_NOP
170# CHECK-NEXT: }
171# CHECK-NEXT: S_WAITCNT 112
172name: bundle_no_waitcnt
173tracksRegLiveness: true
174machineFunctionInfo:
175  isEntryFunction: true
176body: |
177  bb.0:
178    liveins: $vgpr1_vgpr2
179    $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
180    BUNDLE {
181      S_NOP 0
182      S_NOP 0
183    }
184    FLAT_STORE_DWORD $vgpr1_vgpr2, $vgpr0, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
185
186...
187
188---
189
190# See the waitcnt inside the bundle and don't insert an extra
191# CHECK-LABEL: name: preexisting_waitcnt_in_bundle{{$}}
192# CHECK: FLAT_LOAD_DWORD
193# CHECK: S_WAITCNT 0
194# CHECK-NOT: S_WAITCNT
195name: preexisting_waitcnt_in_bundle
196tracksRegLiveness: true
197machineFunctionInfo:
198  isEntryFunction: true
199body: |
200  bb.0:
201    liveins: $vgpr1_vgpr2
202    $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
203    BUNDLE {
204      S_NOP 0
205      S_WAITCNT 0
206    }
207    FLAT_STORE_DWORD $vgpr1_vgpr2, $vgpr0, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
208
209...
210
211---
212
213# Def and use inside bundle
214# CHECK-LABEL: name: insert_in_bundle{{$}}
215# CHECK: BUNDLE implicit-def $vgpr0, implicit $vgpr1_vgpr2 {
216# CHECK-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
217# CHECK-NEXT: S_WAITCNT 112
218# CHECK-NEXT: FLAT_STORE_DWORD $vgpr1_vgpr2, internal $vgpr0, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
219# CHECK-NEXT: }
220
221name: insert_in_bundle
222tracksRegLiveness: true
223machineFunctionInfo:
224  isEntryFunction: true
225body: |
226  bb.0:
227    liveins: $vgpr1_vgpr2
228    BUNDLE implicit-def $vgpr0, implicit $vgpr1_vgpr2 {
229    $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
230    FLAT_STORE_DWORD $vgpr1_vgpr2, internal $vgpr0, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
231    }
232...
233
234---
235
236# Def is last instruction in bundle, use is outside bundle
237
238# CHECK-LABEL: name: exit_bundle{{$}}
239# CHECK: BUNDLE implicit-def $vgpr0, implicit $vgpr1_vgpr2 {
240# CHECK-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
241# CHECK-NEXT: }
242# CHECK-NEXT: S_WAITCNT 112
243# CHECK-NEXT: FLAT_STORE_DWORD $vgpr1_vgpr2, $vgpr0, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
244
245name: exit_bundle
246tracksRegLiveness: true
247machineFunctionInfo:
248  isEntryFunction: true
249body: |
250  bb.0:
251    liveins: $vgpr1_vgpr2
252    BUNDLE implicit-def $vgpr0, implicit $vgpr1_vgpr2 {
253    $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
254    }
255
256    FLAT_STORE_DWORD $vgpr1_vgpr2, $vgpr0, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
257
258...
259
260---
261
262# Def is in bundle, use is in another bundle
263
264# CHECK-LABEL: name: cross_bundle{{$}}
265# CHECK: BUNDLE implicit-def $vgpr0, implicit $vgpr1_vgpr2 {
266# CHECK-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
267# CHECK-NEXT: }
268# CHECK-NEXT: S_WAITCNT 112
269# CHECK-NEXT: BUNDLE implicit $vgpr0, implicit $vgpr1_vgpr2 {
270# CHECK-NEXT: FLAT_STORE_DWORD $vgpr1_vgpr2, $vgpr0, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
271# CHECK-NEXT: }
272
273name: cross_bundle
274tracksRegLiveness: true
275machineFunctionInfo:
276  isEntryFunction: true
277body: |
278  bb.0:
279    liveins: $vgpr1_vgpr2
280    BUNDLE implicit-def $vgpr0, implicit $vgpr1_vgpr2 {
281    $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
282    }
283    BUNDLE implicit $vgpr0, implicit $vgpr1_vgpr2 {
284      FLAT_STORE_DWORD $vgpr1_vgpr2, $vgpr0, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
285    }
286...
287