1; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck  -enable-var-scope -check-prefixes=GCN,CI,MUBUF %s
2; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck  -enable-var-scope -check-prefixes=GCN,GFX9,MUBUF %s
3; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -amdgpu-enable-flat-scratch < %s | FileCheck  -enable-var-scope -check-prefixes=GCN,FLATSCR %s
4
5; GCN-LABEL: {{^}}callee_no_stack:
6; GCN: ; %bb.0:
7; GCN-NEXT: s_waitcnt
8; GCN-NEXT: s_setpc_b64
9define void @callee_no_stack() #0 {
10  ret void
11}
12
13; GCN-LABEL: {{^}}callee_no_stack_no_fp_elim_all:
14; GCN: ; %bb.0:
15; GCN-NEXT: s_waitcnt
16; MUBUF-NEXT:   s_mov_b32 [[FP_COPY:s4]], s33
17; FLATSCR-NEXT: s_mov_b32 [[FP_COPY:s0]], s33
18; GCN-NEXT: s_mov_b32 s33, s32
19; GCN-NEXT: s_mov_b32 s33, [[FP_COPY]]
20; GCN-NEXT: s_setpc_b64
21define void @callee_no_stack_no_fp_elim_all() #1 {
22  ret void
23}
24
25; GCN-LABEL: {{^}}callee_no_stack_no_fp_elim_nonleaf:
26; GCN: ; %bb.0:
27; GCN-NEXT: s_waitcnt
28; GCN-NEXT: s_setpc_b64
29define void @callee_no_stack_no_fp_elim_nonleaf() #2 {
30  ret void
31}
32
33; GCN-LABEL: {{^}}callee_with_stack:
34; GCN: ; %bb.0:
35; GCN-NEXT: s_waitcnt
36; GCN-NEXT: v_mov_b32_e32 v0, 0{{$}}
37; MUBUF-NEXT:   buffer_store_dword v0, off, s[0:3], s32{{$}}
38; FLATSCR-NEXT: scratch_store_dword off, v0, s32
39; GCN-NEXT: s_waitcnt
40; GCN-NEXT: s_setpc_b64
41define void @callee_with_stack() #0 {
42  %alloca = alloca i32, addrspace(5)
43  store volatile i32 0, i32 addrspace(5)* %alloca
44  ret void
45}
46
47; Can use free call clobbered register to preserve original FP value.
48
49; GCN-LABEL: {{^}}callee_with_stack_no_fp_elim_all:
50; GCN: ; %bb.0:
51; GCN-NEXT: s_waitcnt
52; MUBUF-NEXT:   s_mov_b32 [[FP_COPY:s4]], s33
53; FLATSCR-NEXT: s_mov_b32 [[FP_COPY:s0]], s33
54; GCN-NEXT: s_mov_b32 s33, s32
55; MUBUF-NEXT:   s_add_u32 s32, s32, 0x200
56; FLATSCR-NEXT: s_add_u32 s32, s32, 8
57; GCN-NEXT: v_mov_b32_e32 v0, 0{{$}}
58; MUBUF-NEXT:   buffer_store_dword v0, off, s[0:3], s33 offset:4{{$}}
59; FLATSCR-NEXT: scratch_store_dword off, v0, s33 offset:4{{$}}
60; MUBUF-NEXT:   s_sub_u32 s32, s32, 0x200
61; FLATSCR-NEXT: s_sub_u32 s32, s32, 8
62; GCN-NEXT: s_mov_b32 s33, [[FP_COPY]]
63; GCN-NEXT: s_waitcnt vmcnt(0)
64; GCN-NEXT: s_setpc_b64
65define void @callee_with_stack_no_fp_elim_all() #1 {
66  %alloca = alloca i32, addrspace(5)
67  store volatile i32 0, i32 addrspace(5)* %alloca
68  ret void
69}
70
71; GCN-LABEL: {{^}}callee_with_stack_no_fp_elim_non_leaf:
72; GCN: ; %bb.0:
73; GCN-NEXT: s_waitcnt
74; GCN-NEXT: v_mov_b32_e32 v0, 0{{$}}
75; MUBUF-NEXT:   buffer_store_dword v0, off, s[0:3], s32{{$}}
76; FLATSCR-NEXT: scratch_store_dword off, v0, s32{{$}}
77; GCN-NEXT: s_waitcnt
78; GCN-NEXT: s_setpc_b64
79define void @callee_with_stack_no_fp_elim_non_leaf() #2 {
80  %alloca = alloca i32, addrspace(5)
81  store volatile i32 0, i32 addrspace(5)* %alloca
82  ret void
83}
84
85; GCN-LABEL: {{^}}callee_with_stack_and_call:
86; GCN: ; %bb.0:
87; GCN-NEXT: s_waitcnt
88; GCN: s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}}
89; MUBUF-NEXT:   buffer_store_dword [[CSR_VGPR:v[0-9]+]], off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
90; FLATSCR-NEXT: scratch_store_dword off, [[CSR_VGPR:v[0-9]+]], s32 offset:4 ; 4-byte Folded Spill
91; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC0]]
92; GCN: v_writelane_b32 [[CSR_VGPR]], s33, 2
93; GCN-DAG: s_mov_b32 s33, s32
94; MUBUF-DAG:   s_add_u32 s32, s32, 0x400{{$}}
95; FLATSCR-DAG: s_add_u32 s32, s32, 16{{$}}
96; GCN-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}}
97; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s30,
98; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s31,
99
100; MUBUF-DAG:   buffer_store_dword [[ZERO]], off, s[0:3], s33{{$}}
101; FLATSCR-DAG: scratch_store_dword off, [[ZERO]], s33{{$}}
102
103; GCN: s_swappc_b64
104
105; MUBUF-DAG: v_readlane_b32 s5, [[CSR_VGPR]]
106; MUBUF-DAG: v_readlane_b32 s4, [[CSR_VGPR]]
107; FLATSCR-DAG: v_readlane_b32 s0, [[CSR_VGPR]]
108; FLATSCR-DAG: v_readlane_b32 s1, [[CSR_VGPR]]
109
110; MUBUF:    s_sub_u32 s32, s32, 0x400{{$}}
111; FLATSCR:  s_sub_u32 s32, s32, 16{{$}}
112; GCN-NEXT: v_readlane_b32 s33, [[CSR_VGPR]], 2
113; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
114; MUBUF-NEXT:   buffer_load_dword [[CSR_VGPR]], off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
115; FLATSCR-NEXT: scratch_load_dword [[CSR_VGPR]], off, s32 offset:4 ; 4-byte Folded Reload
116; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]]
117; GCN-NEXT: s_waitcnt vmcnt(0)
118
119; GCN-NEXT: s_setpc_b64
120define void @callee_with_stack_and_call() #0 {
121  %alloca = alloca i32, addrspace(5)
122  store volatile i32 0, i32 addrspace(5)* %alloca
123  call void @external_void_func_void()
124  ret void
125}
126
127; Should be able to copy incoming stack pointer directly to inner
128; call's stack pointer argument.
129
130; There is stack usage only because of the need to evict a VGPR for
131; spilling CSR SGPRs.
132
133; GCN-LABEL: {{^}}callee_no_stack_with_call:
134; GCN: s_waitcnt
135; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}}
136; MUBUF-NEXT:   buffer_store_dword [[CSR_VGPR:v[0-9]+]], off, s[0:3], s32 ; 4-byte Folded Spill
137; FLATSCR-NEXT: scratch_store_dword off, [[CSR_VGPR:v[0-9]+]], s32 ; 4-byte Folded Spill
138; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC0]]
139; MUBUF-DAG:   s_add_u32 s32, s32, 0x400
140; FLATSCR-DAG: s_add_u32 s32, s32, 16
141; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s33, [[FP_SPILL_LANE:[0-9]+]]
142
143; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s30, 0
144; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s31, 1
145; GCN: s_swappc_b64
146
147; MUBUF-DAG: v_readlane_b32 s4, v40, 0
148; MUBUF-DAG: v_readlane_b32 s5, v40, 1
149; FLATSCR-DAG: v_readlane_b32 s0, v40, 0
150; FLATSCR-DAG: v_readlane_b32 s1, v40, 1
151
152; MUBUF:   s_sub_u32 s32, s32, 0x400
153; FLATSCR: s_sub_u32 s32, s32, 16
154; GCN-NEXT: v_readlane_b32 s33, [[CSR_VGPR]], [[FP_SPILL_LANE]]
155; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
156; MUBUF-NEXT:   buffer_load_dword [[CSR_VGPR]], off, s[0:3], s32 ; 4-byte Folded Reload
157; FLATSCR-NEXT: scratch_load_dword [[CSR_VGPR]], off, s32 ; 4-byte Folded Reload
158; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]]
159; GCN-NEXT: s_waitcnt vmcnt(0)
160; GCN-NEXT: s_setpc_b64
161define void @callee_no_stack_with_call() #0 {
162  call void @external_void_func_void()
163  ret void
164}
165
166declare hidden void @external_void_func_void() #0
167
168; Make sure if a CSR vgpr is used for SGPR spilling, it is saved and
169; restored. No FP is required.
170;
171; GCN-LABEL: {{^}}callee_func_sgpr_spill_no_calls:
172; GCN: s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}}
173; MUBUF-NEXT:   buffer_store_dword [[CSR_VGPR:v[0-9]+]], off, s[0:3], s32 ; 4-byte Folded Spill
174; FLATSCR-NEXT: scratch_store_dword off, [[CSR_VGPR:v[0-9]+]], s32 ; 4-byte Folded Spill
175; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC0]]
176; GCN: v_writelane_b32 [[CSR_VGPR]], s
177; GCN: v_writelane_b32 [[CSR_VGPR]], s
178
179; GCN: ;;#ASMSTART
180; GCN: v_readlane_b32 s{{[0-9]+}}, [[CSR_VGPR]]
181; GCN: v_readlane_b32 s{{[0-9]+}}, [[CSR_VGPR]]
182
183; GCN: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
184; MUBUF-NEXT:   buffer_load_dword [[CSR_VGPR]], off, s[0:3], s32 ; 4-byte Folded Reload
185; FLATSCR-NEXT: scratch_load_dword [[CSR_VGPR]], off, s32 ; 4-byte Folded Reload
186; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]]
187; GCN-NEXT: s_waitcnt
188; GCN-NEXT: s_setpc_b64
189define void @callee_func_sgpr_spill_no_calls(i32 %in) #0 {
190  call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7}"() #0
191  call void asm sideeffect "", "~{v8},~{v9},~{v10},~{v11},~{v12},~{v13},~{v14},~{v15}"() #0
192  call void asm sideeffect "", "~{v16},~{v17},~{v18},~{v19},~{v20},~{v21},~{v22},~{v23}"() #0
193  call void asm sideeffect "", "~{v24},~{v25},~{v26},~{v27},~{v28},~{v29},~{v30},~{v31}"() #0
194  call void asm sideeffect "", "~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38},~{v39}"() #0
195
196  %wide.sgpr0 = call <16 x i32> asm sideeffect "; def $0", "=s" () #0
197  %wide.sgpr1 = call <16 x i32> asm sideeffect "; def $0", "=s" () #0
198  %wide.sgpr2 = call <16 x i32> asm sideeffect "; def $0", "=s" () #0
199  %wide.sgpr5 = call <16 x i32> asm sideeffect "; def $0", "=s" () #0
200  %wide.sgpr3 = call <8 x i32> asm sideeffect "; def $0", "=s" () #0
201  %wide.sgpr4 = call <2 x i32> asm sideeffect "; def $0", "=s" () #0
202
203  call void asm sideeffect "; use $0", "s"(<16 x i32> %wide.sgpr0) #0
204  call void asm sideeffect "; use $0", "s"(<16 x i32> %wide.sgpr1) #0
205  call void asm sideeffect "; use $0", "s"(<16 x i32> %wide.sgpr2) #0
206  call void asm sideeffect "; use $0", "s"(<8 x i32> %wide.sgpr3) #0
207  call void asm sideeffect "; use $0", "s"(<2 x i32> %wide.sgpr4) #0
208  call void asm sideeffect "; use $0", "s"(<16 x i32> %wide.sgpr5) #0
209  ret void
210}
211
212; Has no spilled CSR VGPRs used for SGPR spilling, so no need to
213; enable all lanes and restore.
214
215; GCN-LABEL: {{^}}spill_only_csr_sgpr:
216; GCN: s_waitcnt
217; GCN-NEXT: v_writelane_b32 v0, s42, 0
218; GCN-NEXT: ;;#ASMSTART
219; GCN-NEXT: ; clobber s42
220; GCN-NEXT: ;;#ASMEND
221; GCN-NEXT: v_readlane_b32 s42, v0, 0
222; GCN-NEXT: s_setpc_b64
223define void @spill_only_csr_sgpr() {
224  call void asm sideeffect "; clobber s42", "~{s42}"()
225  ret void
226}
227
228; TODO: Can the SP inc/deec be remvoed?
229; GCN-LABEL: {{^}}callee_with_stack_no_fp_elim_csr_vgpr:
230; GCN: s_waitcnt
231; GCN-NEXT:s_mov_b32 [[FP_COPY:s[0-9]+]], s33
232; GCN-NEXT: s_mov_b32 s33, s32
233; GCN: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0
234; MUBUF-DAG:   buffer_store_dword v41, off, s[0:3], s33 ; 4-byte Folded Spill
235; FLATSCR-DAG: scratch_store_dword off, v41, s33 ; 4-byte Folded Spill
236; MUBUF-DAG:   buffer_store_dword [[ZERO]], off, s[0:3], s33 offset:8
237; FLATSCR-DAG: scratch_store_dword off, [[ZERO]], s33 offset:8
238
239; GCN:	;;#ASMSTART
240; GCN-NEXT: ; clobber v41
241; GCN-NEXT: ;;#ASMEND
242
243; MUBUF:   buffer_load_dword v41, off, s[0:3], s33 ; 4-byte Folded Reload
244; FLATSCR: scratch_load_dword v41, off, s33 ; 4-byte Folded Reload
245; MUBUF:        s_add_u32 s32, s32, 0x300
246; MUBUF-NEXT:   s_sub_u32 s32, s32, 0x300
247; MUBUF-NEXT:   s_mov_b32 s33, s4
248; FLATSCR:      s_add_u32 s32, s32, 12
249; FLATSCR-NEXT: s_sub_u32 s32, s32, 12
250; FLATSCR-NEXT: s_mov_b32 s33, s0
251; GCN-NEXT: s_waitcnt vmcnt(0)
252; GCN-NEXT: s_setpc_b64
253define void @callee_with_stack_no_fp_elim_csr_vgpr() #1 {
254  %alloca = alloca i32, addrspace(5)
255  store volatile i32 0, i32 addrspace(5)* %alloca
256  call void asm sideeffect "; clobber v41", "~{v41}"()
257  ret void
258}
259
260; Use a copy to a free SGPR instead of introducing a second CSR VGPR.
261; GCN-LABEL: {{^}}last_lane_vgpr_for_fp_csr:
262; GCN: s_waitcnt
263; GCN-NEXT: v_writelane_b32 v1, s33, 63
264; GCN-COUNT-60: v_writelane_b32 v1
265; GCN: s_mov_b32 s33, s32
266; GCN-COUNT-2: v_writelane_b32 v1
267; MUBUF:   buffer_store_dword v41, off, s[0:3], s33 ; 4-byte Folded Spill
268; FLATSCR: scratch_store_dword off, v41, s33 ; 4-byte Folded Spill
269; MUBUF:   buffer_store_dword v{{[0-9]+}}, off, s[0:3], s33 offset:8
270; FLATSCR: scratch_store_dword off, v{{[0-9]+}}, s33 offset:8
271; GCN: ;;#ASMSTART
272; GCN: v_writelane_b32 v1
273
274; MUBUF:        s_add_u32 s32, s32, 0x300
275; MUBUF:        s_sub_u32 s32, s32, 0x300
276; FLATSCR:      s_add_u32 s32, s32, 12
277; FLATSCR:      s_sub_u32 s32, s32, 12
278; GCN-NEXT: v_readlane_b32 s33, v1, 63
279; GCN-NEXT: s_waitcnt vmcnt(0)
280; GCN-NEXT: s_setpc_b64
281define void @last_lane_vgpr_for_fp_csr() #1 {
282  %alloca = alloca i32, addrspace(5)
283  store volatile i32 0, i32 addrspace(5)* %alloca
284  call void asm sideeffect "; clobber v41", "~{v41}"()
285  call void asm sideeffect "",
286    "~{s40},~{s41},~{s42},~{s43},~{s44},~{s45},~{s46},~{s47},~{s48},~{s49}
287    ,~{s50},~{s51},~{s52},~{s53},~{s54},~{s55},~{s56},~{s57},~{s58},~{s59}
288    ,~{s60},~{s61},~{s62},~{s63},~{s64},~{s65},~{s66},~{s67},~{s68},~{s69}
289    ,~{s70},~{s71},~{s72},~{s73},~{s74},~{s75},~{s76},~{s77},~{s78},~{s79}
290    ,~{s80},~{s81},~{s82},~{s83},~{s84},~{s85},~{s86},~{s87},~{s88},~{s89}
291    ,~{s90},~{s91},~{s92},~{s93},~{s94},~{s95},~{s96},~{s97},~{s98},~{s99}
292    ,~{s100},~{s101},~{s102}"() #1
293
294  ret void
295}
296
297; Use a copy to a free SGPR instead of introducing a second CSR VGPR.
298; GCN-LABEL: {{^}}no_new_vgpr_for_fp_csr:
299; GCN: s_waitcnt
300; GCN-COUNT-62: v_writelane_b32 v1,
301; GCN: s_mov_b32 [[FP_COPY:s[0-9]+]], s33
302; GCN-NEXT: s_mov_b32 s33, s32
303; GCN: v_writelane_b32 v1,
304; MUBUF:   buffer_store_dword v41, off, s[0:3], s33 ; 4-byte Folded Spill
305; FLATSCR: scratch_store_dword off, v41, s33 ; 4-byte Folded Spill
306; MUBUF:   buffer_store_dword
307; FLATSCR: scratch_store_dword
308; GCN: ;;#ASMSTART
309; GCN: v_writelane_b32 v1,
310; MUBUF:   buffer_load_dword v41, off, s[0:3], s33 ; 4-byte Folded Reload
311; FLATSCR: scratch_load_dword v41, off, s33 ; 4-byte Folded Reload
312; MUBUF:        s_add_u32 s32, s32, 0x300
313; FLATSCR:      s_add_u32 s32, s32, 12
314; GCN-COUNT-64: v_readlane_b32 s{{[0-9]+}}, v1
315; MUBUF-NEXT:   s_sub_u32 s32, s32, 0x300
316; FLATSCR-NEXT: s_sub_u32 s32, s32, 12
317; GCN-NEXT: s_mov_b32 s33, [[FP_COPY]]
318; GCN-NEXT: s_waitcnt vmcnt(0)
319; GCN-NEXT: s_setpc_b64
320define void @no_new_vgpr_for_fp_csr() #1 {
321  %alloca = alloca i32, addrspace(5)
322  store volatile i32 0, i32 addrspace(5)* %alloca
323  call void asm sideeffect "; clobber v41", "~{v41}"()
324  call void asm sideeffect "",
325    "~{s39},~{s40},~{s41},~{s42},~{s43},~{s44},~{s45},~{s46},~{s47},~{s48},~{s49}
326    ,~{s50},~{s51},~{s52},~{s53},~{s54},~{s55},~{s56},~{s57},~{s58},~{s59}
327    ,~{s60},~{s61},~{s62},~{s63},~{s64},~{s65},~{s66},~{s67},~{s68},~{s69}
328    ,~{s70},~{s71},~{s72},~{s73},~{s74},~{s75},~{s76},~{s77},~{s78},~{s79}
329    ,~{s80},~{s81},~{s82},~{s83},~{s84},~{s85},~{s86},~{s87},~{s88},~{s89}
330    ,~{s90},~{s91},~{s92},~{s93},~{s94},~{s95},~{s96},~{s97},~{s98},~{s99}
331    ,~{s100},~{s101},~{s102}"() #1
332
333  ret void
334}
335
336; GCN-LABEL: {{^}}realign_stack_no_fp_elim:
337; GCN: s_waitcnt
338; MUBUF-NEXT:   s_add_u32 [[SCRATCH:s[0-9]+]], s32, 0x7ffc0
339; FLATSCR-NEXT: s_add_u32 [[SCRATCH:s[0-9]+]], s32, 0x1fff
340; MUBUF-NEXT:   s_mov_b32 [[FP_COPY:s4]], s33
341; FLATSCR-NEXT: s_mov_b32 [[FP_COPY:s0]], s33
342; MUBUF-NEXT:   s_and_b32 s33, [[SCRATCH]], 0xfff80000
343; FLATSCR-NEXT: s_and_b32 s33, [[SCRATCH]], 0xffffe000
344; MUBUF-NEXT:   s_add_u32 s32, s32, 0x100000
345; FLATSCR-NEXT: s_add_u32 s32, s32, 0x4000
346; GCN-NEXT:     v_mov_b32_e32 [[ZERO:v[0-9]+]], 0
347; MUBUF-NEXT:   buffer_store_dword [[ZERO]], off, s[0:3], s33
348; FLATSCR-NEXT: scratch_store_dword off, [[ZERO]], s33
349; MUBUF-NEXT:   s_sub_u32 s32, s32, 0x100000
350; FLATSCR-NEXT: s_sub_u32 s32, s32, 0x4000
351; GCN-NEXT: s_mov_b32 s33, [[FP_COPY]]
352; GCN-NEXT: s_waitcnt vmcnt(0)
353; GCN-NEXT: s_setpc_b64
354define void @realign_stack_no_fp_elim() #1 {
355  %alloca = alloca i32, align 8192, addrspace(5)
356  store volatile i32 0, i32 addrspace(5)* %alloca
357  ret void
358}
359
360; GCN-LABEL: {{^}}no_unused_non_csr_sgpr_for_fp:
361; GCN: s_waitcnt
362; GCN-NEXT: v_writelane_b32 v1, s33, 2
363; GCN-NEXT: v_writelane_b32 v1, s30, 0
364; GCN-NEXT: s_mov_b32 s33, s32
365; GCN: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0
366; GCN: v_writelane_b32 v1, s31, 1
367; MUBUF:   buffer_store_dword [[ZERO]], off, s[0:3], s33 offset:4
368; FLATSCR: scratch_store_dword off, [[ZERO]], s33 offset:4
369; GCN: ;;#ASMSTART
370; MUBUF:        v_readlane_b32 s4, v1, 0
371; MUBUF-NEXT:   s_add_u32 s32, s32, 0x200
372; MUBUF-NEXT:   v_readlane_b32 s5, v1, 1
373; FLATSCR:      v_readlane_b32 s0, v1, 0
374; FLATSCR-NEXT: s_add_u32 s32, s32, 8
375; FLATSCR-NEXT: v_readlane_b32 s1, v1, 1
376; MUBUF-NEXT:   s_sub_u32 s32, s32, 0x200
377; FLATSCR-NEXT: s_sub_u32 s32, s32, 8
378; GCN-NEXT:     v_readlane_b32 s33, v1, 2
379; GCN-NEXT:     s_waitcnt vmcnt(0)
380; MUBUF-NEXT:   s_setpc_b64 s[4:5]
381; FLATSCR-NEXT: s_setpc_b64 s[0:1]
382define void @no_unused_non_csr_sgpr_for_fp() #1 {
383  %alloca = alloca i32, addrspace(5)
384  store volatile i32 0, i32 addrspace(5)* %alloca
385
386  ; Use all clobberable registers, so FP has to spill to a VGPR.
387  call void asm sideeffect "",
388    "~{s0},~{s1},~{s2},~{s3},~{s4},~{s5},~{s6},~{s7},~{s8},~{s9}
389    ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19}
390    ,~{s20},~{s21},~{s22},~{s23},~{s24},~{s25},~{s26},~{s27},~{s28},~{s29}
391    ,~{s30},~{s31}"() #0
392
393  ret void
394}
395
396; Need a new CSR VGPR to satisfy the FP spill.
397; GCN-LABEL: {{^}}no_unused_non_csr_sgpr_for_fp_no_scratch_vgpr:
398; GCN: s_waitcnt
399; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}}
400; MUBUF-NEXT:   buffer_store_dword [[CSR_VGPR:v[0-9]+]], off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
401; FLATSCR-NEXT: scratch_store_dword off, [[CSR_VGPR:v[0-9]+]], s32 offset:8 ; 4-byte Folded Spill
402; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC0]]
403; GCN-NEXT: v_writelane_b32 [[CSR_VGPR]], s33, 2
404; GCN-NEXT: v_writelane_b32 [[CSR_VGPR]], s30, 0
405; GCN-NEXT: s_mov_b32 s33, s32
406
407; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s31, 1
408; MUBUF-DAG:   buffer_store_dword
409; FLATSCR-DAG: scratch_store_dword
410; MUBUF:       s_add_u32 s32, s32, 0x300{{$}}
411; FLATSCR:     s_add_u32 s32, s32, 12{{$}}
412
413; MUBUF:        v_readlane_b32 s4, [[CSR_VGPR]], 0
414; FLATSCR:      v_readlane_b32 s0, [[CSR_VGPR]], 0
415; GCN: ;;#ASMSTART
416; MUBUF:        v_readlane_b32 s5, [[CSR_VGPR]], 1
417; FLATSCR:      v_readlane_b32 s1, [[CSR_VGPR]], 1
418; MUBUF-NEXT:   s_sub_u32 s32, s32, 0x300{{$}}
419; FLATSCR-NEXT: s_sub_u32 s32, s32, 12{{$}}
420; GCN-NEXT: v_readlane_b32 s33, [[CSR_VGPR]], 2
421; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
422; MUBUF-NEXT:   buffer_load_dword [[CSR_VGPR]], off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
423; FLATSCR-NEXT: scratch_load_dword [[CSR_VGPR]], off, s32 offset:8 ; 4-byte Folded Reload
424; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]]
425; GCN-NEXT: s_waitcnt vmcnt(0)
426; GCN-NEXT: s_setpc_b64
427define void @no_unused_non_csr_sgpr_for_fp_no_scratch_vgpr() #1 {
428  %alloca = alloca i32, addrspace(5)
429  store volatile i32 0, i32 addrspace(5)* %alloca
430
431  ; Use all clobberable registers, so FP has to spill to a VGPR.
432  call void asm sideeffect "",
433    "~{s0},~{s1},~{s2},~{s3},~{s4},~{s5},~{s6},~{s7},~{s8},~{s9}
434    ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19}
435    ,~{s20},~{s21},~{s22},~{s23},~{s24},~{s25},~{s26},~{s27},~{s28},~{s29}
436    ,~{s30},~{s31}"() #0
437
438  call void asm sideeffect "; clobber nonpreserved initial VGPRs",
439    "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9}
440    ,~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19}
441    ,~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29}
442    ,~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38},~{v39}"() #1
443
444  ret void
445}
446
447; The byval argument exceeds the MUBUF constant offset, so a scratch
448; register is needed to access the CSR VGPR slot.
449; GCN-LABEL: {{^}}scratch_reg_needed_mubuf_offset:
450; GCN: s_waitcnt
451; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}}
452; MUBUF-NEXT: v_mov_b32_e32 [[SCRATCH_VGPR:v[0-9]+]], 0x1008
453; MUBUF-NEXT: buffer_store_dword [[CSR_VGPR:v[0-9]+]], [[SCRATCH_VGPR]], s[0:3], s32 offen ; 4-byte Folded Spill
454; FLATSCR-NEXT: s_add_u32 [[SCRATCH_SGPR:s[0-9]+]], s32, 0x1008
455; FLATSCR-NEXT: scratch_store_dword off, [[CSR_VGPR:v[0-9]+]], [[SCRATCH_SGPR]] ; 4-byte Folded Spill
456; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC0]]
457; GCN-NEXT: v_writelane_b32 [[CSR_VGPR]], s33, 2
458; GCN-DAG:  v_writelane_b32 [[CSR_VGPR]], s30, 0
459; GCN-DAG:  s_mov_b32 s33, s32
460; GCN-DAG:  v_writelane_b32 [[CSR_VGPR]], s31, 1
461; MUBUF-DAG:   s_add_u32 s32, s32, 0x40300{{$}}
462; FLATSCR-DAG: s_add_u32 s32, s32, 0x100c{{$}}
463; MUBUF-DAG:   buffer_store_dword
464; FLATSCR-DAG: scratch_store_dword
465
466; MUBUF:   v_readlane_b32 s4, [[CSR_VGPR]], 0
467; FLATSCR: v_readlane_b32 s0, [[CSR_VGPR]], 0
468; GCN: ;;#ASMSTART
469; MUBUF:   v_readlane_b32 s5, [[CSR_VGPR]], 1
470; FLATSCR: v_readlane_b32 s1, [[CSR_VGPR]], 1
471; MUBUF-NEXT:   s_sub_u32 s32, s32, 0x40300{{$}}
472; FLATSCR-NEXT: s_sub_u32 s32, s32, 0x100c{{$}}
473; GCN-NEXT: v_readlane_b32 s33, [[CSR_VGPR]], 2
474; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
475; MUBUF-NEXT: v_mov_b32_e32 [[SCRATCH_VGPR:v[0-9]+]], 0x1008
476; MUBUF-NEXT: buffer_load_dword [[CSR_VGPR]], [[SCRATCH_VGPR]], s[0:3], s32 offen ; 4-byte Folded Reload
477; FLATSCR-NEXT: s_add_u32 [[SCRATCH_SGPR:s[0-9]+]], s32, 0x1008
478; FLATSCR-NEXT: scratch_load_dword [[CSR_VGPR]], off, [[SCRATCH_SGPR]] ; 4-byte Folded Reload
479; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]]
480; GCN-NEXT: s_waitcnt vmcnt(0)
481; GCN-NEXT: s_setpc_b64
482define void @scratch_reg_needed_mubuf_offset([4096 x i8] addrspace(5)* byval align 4 %arg) #1 {
483  %alloca = alloca i32, addrspace(5)
484  store volatile i32 0, i32 addrspace(5)* %alloca
485
486  ; Use all clobberable registers, so FP has to spill to a VGPR.
487  call void asm sideeffect "; clobber nonpreserved SGPRs",
488    "~{s0},~{s1},~{s2},~{s3},~{s4},~{s5},~{s6},~{s7},~{s8},~{s9}
489    ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19}
490    ,~{s20},~{s21},~{s22},~{s23},~{s24},~{s25},~{s26},~{s27},~{s28},~{s29}
491    ,~{s30},~{s31}"() #0
492
493  ; Use all clobberable VGPRs, so a CSR spill is needed for the VGPR
494  call void asm sideeffect "; clobber nonpreserved VGPRs",
495    "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9}
496    ,~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19}
497    ,~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29}
498    ,~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38},~{v39}"() #1
499
500  ret void
501}
502
503; GCN-LABEL: {{^}}local_empty_func:
504; GCN: s_waitcnt
505; GCN-NEXT: s_setpc_b64
506define internal void @local_empty_func() #0 {
507  ret void
508}
509
510; An FP is needed, despite not needing any spills
511; TODO: Ccould see callee does not use stack and omit FP.
512; GCN-LABEL: {{^}}ipra_call_with_stack:
513; GCN: s_mov_b32 [[FP_COPY:s[0-9]+]], s33
514; GCN: s_mov_b32 s33, s32
515; MUBUF:   s_add_u32 s32, s32, 0x400
516; FLATSCR: s_add_u32 s32, s32, 16
517; MUBUF:   buffer_store_dword v{{[0-9]+}}, off, s[0:3], s33{{$}}
518; FLATSCR: scratch_store_dword off, v{{[0-9]+}}, s33{{$}}
519; GCN:     s_swappc_b64
520; MUBUF:   s_sub_u32 s32, s32, 0x400
521; FLATSCR: s_sub_u32 s32, s32, 16
522; GCN: s_mov_b32 s33, [[FP_COPY:s[0-9]+]]
523define void @ipra_call_with_stack() #0 {
524  %alloca = alloca i32, addrspace(5)
525  store volatile i32 0, i32 addrspace(5)* %alloca
526  call void @local_empty_func()
527  ret void
528}
529
530; With no free registers, we must spill the FP to memory.
531; GCN-LABEL: {{^}}callee_need_to_spill_fp_to_memory:
532; MUBUF:   s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
533; MUBUF:   v_mov_b32_e32 [[TMP_VGPR1:v[0-9]+]], s33
534; MUBUF:   buffer_store_dword [[TMP_VGPR1]], off, s[0:3], s32 offset:4
535; MUBUF:   s_mov_b64 exec, [[COPY_EXEC1]]
536; FLATSCR: s_mov_b32 s0, s33
537; GCN:     s_mov_b32 s33, s32
538; MUBUF:   s_or_saveexec_b64 [[COPY_EXEC2:s\[[0-9]+:[0-9]+\]]], -1{{$}}
539; MUBUF:   buffer_load_dword [[TMP_VGPR2:v[0-9]+]], off, s[0:3], s32 offset:4
540; FLATSCR: s_mov_b32 s33, s0
541; MUBUF:   s_waitcnt vmcnt(0)
542; MUBUF:   v_readfirstlane_b32 s33, [[TMP_VGPR2]]
543; MUBUF:   s_mov_b64 exec, [[COPY_EXEC2]]
544; GCN:     s_setpc_b64
545; MUBUF:   ScratchSize: 8
546; FLATSCR: ScratchSize: 0
547define void @callee_need_to_spill_fp_to_memory() #3 {
548  call void asm sideeffect "; clobber nonpreserved SGPRs",
549    "~{s4},~{s5},~{s6},~{s7},~{s8},~{s9}
550    ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19}
551    ,~{s20},~{s21},~{s22},~{s23},~{s24},~{s25},~{s26},~{s27},~{s28},~{s29}
552    ,~{vcc}"()
553
554  call void asm sideeffect "; clobber all VGPRs",
555    "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9}
556    ,~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19}
557    ,~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29}
558    ,~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38},~{v39}"()
559  ret void
560}
561
562; If we have a reserved VGPR that can be used for SGPR spills, we may still
563; need to spill the FP to memory if there are no free lanes in the reserved
564; VGPR.
565; GCN-LABEL: {{^}}callee_need_to_spill_fp_to_memory_full_reserved_vgpr:
566; MUBUF:   s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
567; MUBUF:   v_mov_b32_e32 [[TMP_VGPR1:v[0-9]+]], s33
568; MUBUF:   buffer_store_dword [[TMP_VGPR1]], off, s[0:3], s32 offset:[[OFF:[0-9]+]]
569; MUBUF:   s_mov_b64 exec, [[COPY_EXEC1]]
570; GCN-NOT: v_writelane_b32 v40, s33
571; MUBUF:   s_mov_b32 s33, s32
572; FLATSCR: s_mov_b32 s33, s0
573; GCN-NOT: v_readlane_b32 s33, v40
574; MUBUF:   s_or_saveexec_b64 [[COPY_EXEC2:s\[[0-9]+:[0-9]+\]]], -1{{$}}
575; MUBUF:   buffer_load_dword [[TMP_VGPR2:v[0-9]+]], off, s[0:3], s32 offset:[[OFF]]
576; MUBUF:   v_readfirstlane_b32 s33, [[TMP_VGPR2]]
577; MUBUF:   s_mov_b64 exec, [[COPY_EXEC2]]
578; GCN:     s_setpc_b64
579define void @callee_need_to_spill_fp_to_memory_full_reserved_vgpr() #3 {
580  call void asm sideeffect "; clobber nonpreserved SGPRs and 64 CSRs",
581    "~{s4},~{s5},~{s6},~{s7},~{s8},~{s9}
582    ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19}
583    ,~{s20},~{s21},~{s22},~{s23},~{s24},~{s25},~{s26},~{s27},~{s28},~{s29}
584    ,~{s40},~{s41},~{s42},~{s43},~{s44},~{s45},~{s46},~{s47},~{s48},~{s49}
585    ,~{s50},~{s51},~{s52},~{s53},~{s54},~{s55},~{s56},~{s57},~{s58},~{s59}
586    ,~{s60},~{s61},~{s62},~{s63},~{s64},~{s65},~{s66},~{s67},~{s68},~{s69}
587    ,~{s70},~{s71},~{s72},~{s73},~{s74},~{s75},~{s76},~{s77},~{s78},~{s79}
588    ,~{s80},~{s81},~{s82},~{s83},~{s84},~{s85},~{s86},~{s87},~{s88},~{s89}
589    ,~{s90},~{s91},~{s92},~{s93},~{s94},~{s95},~{s96},~{s97},~{s98},~{s99}
590    ,~{s100},~{s101},~{s102},~{s39},~{vcc}"()
591
592  call void asm sideeffect "; clobber all VGPRs except CSR v40",
593    "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9}
594    ,~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19}
595    ,~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29}
596    ,~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38}"()
597  ret void
598}
599
600; If the size of the offset exceeds the MUBUF offset field we need another
601; scratch VGPR to hold the offset.
602; GCN-LABEL: {{^}}spill_fp_to_memory_scratch_reg_needed_mubuf_offset
603; MUBUF: s_or_saveexec_b64 s[4:5], -1
604; MUBUF: v_mov_b32_e32 v0, s33
605; GCN-NOT: v_mov_b32_e32 v0, 0x1008
606; MUBUF-NEXT: v_mov_b32_e32 v1, 0x1008
607; MUBUF-NEXT: buffer_store_dword v0, v1, s[0:3], s32 offen ; 4-byte Folded Spill
608; FLATSCR: s_add_u32 [[SOFF:s[0-9]+]], s33, 0x1004
609; FLATSCR: v_mov_b32_e32 v0, 0
610; FLATSCR: scratch_store_dword off, v0, [[SOFF]]
611define void @spill_fp_to_memory_scratch_reg_needed_mubuf_offset([4096 x i8] addrspace(5)* byval align 4 %arg) #3 {
612  %alloca = alloca i32, addrspace(5)
613  store volatile i32 0, i32 addrspace(5)* %alloca
614
615  call void asm sideeffect "; clobber nonpreserved SGPRs and 64 CSRs",
616    "~{s4},~{s5},~{s6},~{s7},~{s8},~{s9}
617    ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19}
618    ,~{s20},~{s21},~{s22},~{s23},~{s24},~{s25},~{s26},~{s27},~{s28},~{s29}
619    ,~{s40},~{s41},~{s42},~{s43},~{s44},~{s45},~{s46},~{s47},~{s48},~{s49}
620    ,~{s50},~{s51},~{s52},~{s53},~{s54},~{s55},~{s56},~{s57},~{s58},~{s59}
621    ,~{s60},~{s61},~{s62},~{s63},~{s64},~{s65},~{s66},~{s67},~{s68},~{s69}
622    ,~{s70},~{s71},~{s72},~{s73},~{s74},~{s75},~{s76},~{s77},~{s78},~{s79}
623    ,~{s80},~{s81},~{s82},~{s83},~{s84},~{s85},~{s86},~{s87},~{s88},~{s89}
624    ,~{s90},~{s91},~{s92},~{s93},~{s94},~{s95},~{s96},~{s97},~{s98},~{s99}
625    ,~{s100},~{s101},~{s102},~{s39},~{vcc}"()
626
627  call void asm sideeffect "; clobber all VGPRs except CSR v40",
628    "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9}
629    ,~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19}
630    ,~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29}
631    ,~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38}"()
632  ret void
633}
634
635attributes #0 = { nounwind }
636attributes #1 = { nounwind "frame-pointer"="all" }
637attributes #2 = { nounwind "frame-pointer"="non-leaf" }
638attributes #3 = { nounwind "frame-pointer"="all" "amdgpu-waves-per-eu"="6,6" }
639