1; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,MUBUF %s 2; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,MUBUF %s 3; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -amdgpu-enable-flat-scratch < %s | FileCheck -enable-var-scope -check-prefixes=GCN,FLATSCR %s 4 5; GCN-LABEL: {{^}}callee_no_stack: 6; GCN: ; %bb.0: 7; GCN-NEXT: s_waitcnt 8; GCN-NEXT: s_setpc_b64 9define void @callee_no_stack() #0 { 10 ret void 11} 12 13; GCN-LABEL: {{^}}callee_no_stack_no_fp_elim_all: 14; GCN: ; %bb.0: 15; GCN-NEXT: s_waitcnt 16; MUBUF-NEXT: s_mov_b32 [[FP_COPY:s4]], s33 17; FLATSCR-NEXT: s_mov_b32 [[FP_COPY:s0]], s33 18; GCN-NEXT: s_mov_b32 s33, s32 19; GCN-NEXT: s_mov_b32 s33, [[FP_COPY]] 20; GCN-NEXT: s_setpc_b64 21define void @callee_no_stack_no_fp_elim_all() #1 { 22 ret void 23} 24 25; GCN-LABEL: {{^}}callee_no_stack_no_fp_elim_nonleaf: 26; GCN: ; %bb.0: 27; GCN-NEXT: s_waitcnt 28; GCN-NEXT: s_setpc_b64 29define void @callee_no_stack_no_fp_elim_nonleaf() #2 { 30 ret void 31} 32 33; GCN-LABEL: {{^}}callee_with_stack: 34; GCN: ; %bb.0: 35; GCN-NEXT: s_waitcnt 36; GCN-NEXT: v_mov_b32_e32 v0, 0{{$}} 37; MUBUF-NEXT: buffer_store_dword v0, off, s[0:3], s32{{$}} 38; FLATSCR-NEXT: scratch_store_dword off, v0, s32 39; GCN-NEXT: s_waitcnt 40; GCN-NEXT: s_setpc_b64 41define void @callee_with_stack() #0 { 42 %alloca = alloca i32, addrspace(5) 43 store volatile i32 0, i32 addrspace(5)* %alloca 44 ret void 45} 46 47; Can use free call clobbered register to preserve original FP value. 48 49; GCN-LABEL: {{^}}callee_with_stack_no_fp_elim_all: 50; GCN: ; %bb.0: 51; GCN-NEXT: s_waitcnt 52; MUBUF-NEXT: s_mov_b32 [[FP_COPY:s4]], s33 53; FLATSCR-NEXT: s_mov_b32 [[FP_COPY:s0]], s33 54; GCN-NEXT: s_mov_b32 s33, s32 55; MUBUF-NEXT: s_add_u32 s32, s32, 0x200 56; FLATSCR-NEXT: s_add_u32 s32, s32, 8 57; GCN-NEXT: v_mov_b32_e32 v0, 0{{$}} 58; MUBUF-NEXT: buffer_store_dword v0, off, s[0:3], s33 offset:4{{$}} 59; FLATSCR-NEXT: scratch_store_dword off, v0, s33 offset:4{{$}} 60; GCN-NEXT: s_waitcnt vmcnt(0) 61; MUBUF-NEXT: s_sub_u32 s32, s32, 0x200 62; FLATSCR-NEXT: s_sub_u32 s32, s32, 8 63; GCN-NEXT: s_mov_b32 s33, [[FP_COPY]] 64; GCN-NEXT: s_setpc_b64 65define void @callee_with_stack_no_fp_elim_all() #1 { 66 %alloca = alloca i32, addrspace(5) 67 store volatile i32 0, i32 addrspace(5)* %alloca 68 ret void 69} 70 71; GCN-LABEL: {{^}}callee_with_stack_no_fp_elim_non_leaf: 72; GCN: ; %bb.0: 73; GCN-NEXT: s_waitcnt 74; GCN-NEXT: v_mov_b32_e32 v0, 0{{$}} 75; MUBUF-NEXT: buffer_store_dword v0, off, s[0:3], s32{{$}} 76; FLATSCR-NEXT: scratch_store_dword off, v0, s32{{$}} 77; GCN-NEXT: s_waitcnt 78; GCN-NEXT: s_setpc_b64 79define void @callee_with_stack_no_fp_elim_non_leaf() #2 { 80 %alloca = alloca i32, addrspace(5) 81 store volatile i32 0, i32 addrspace(5)* %alloca 82 ret void 83} 84 85; GCN-LABEL: {{^}}callee_with_stack_and_call: 86; GCN: ; %bb.0: 87; GCN-NEXT: s_waitcnt 88; GCN: s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}} 89; MUBUF-NEXT: buffer_store_dword [[CSR_VGPR:v[0-9]+]], off, s[0:3], s32 offset:4 ; 4-byte Folded Spill 90; FLATSCR-NEXT: scratch_store_dword off, [[CSR_VGPR:v[0-9]+]], s32 offset:4 ; 4-byte Folded Spill 91; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC0]] 92; GCN: v_writelane_b32 [[CSR_VGPR]], s33, 2 93; GCN-DAG: s_mov_b32 s33, s32 94; MUBUF-DAG: s_add_u32 s32, s32, 0x400{{$}} 95; FLATSCR-DAG: s_add_u32 s32, s32, 16{{$}} 96; GCN-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}} 97; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s30, 98; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s31, 99 100; MUBUF-DAG: buffer_store_dword [[ZERO]], off, s[0:3], s33{{$}} 101; FLATSCR-DAG: scratch_store_dword off, [[ZERO]], s33{{$}} 102 103; GCN: s_swappc_b64 104 105; MUBUF-DAG: v_readlane_b32 s5, [[CSR_VGPR]] 106; MUBUF-DAG: v_readlane_b32 s4, [[CSR_VGPR]] 107; FLATSCR-DAG: v_readlane_b32 s0, [[CSR_VGPR]] 108; FLATSCR-DAG: v_readlane_b32 s1, [[CSR_VGPR]] 109 110; MUBUF: s_sub_u32 s32, s32, 0x400{{$}} 111; FLATSCR: s_sub_u32 s32, s32, 16{{$}} 112; GCN-NEXT: v_readlane_b32 s33, [[CSR_VGPR]], 2 113; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}} 114; MUBUF-NEXT: buffer_load_dword [[CSR_VGPR]], off, s[0:3], s32 offset:4 ; 4-byte Folded Reload 115; FLATSCR-NEXT: scratch_load_dword [[CSR_VGPR]], off, s32 offset:4 ; 4-byte Folded Reload 116; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]] 117; GCN-NEXT: s_waitcnt vmcnt(0) 118 119; GCN-NEXT: s_setpc_b64 120define void @callee_with_stack_and_call() #0 { 121 %alloca = alloca i32, addrspace(5) 122 store volatile i32 0, i32 addrspace(5)* %alloca 123 call void @external_void_func_void() 124 ret void 125} 126 127; Should be able to copy incoming stack pointer directly to inner 128; call's stack pointer argument. 129 130; There is stack usage only because of the need to evict a VGPR for 131; spilling CSR SGPRs. 132 133; GCN-LABEL: {{^}}callee_no_stack_with_call: 134; GCN: s_waitcnt 135; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}} 136; MUBUF-NEXT: buffer_store_dword [[CSR_VGPR:v[0-9]+]], off, s[0:3], s32 ; 4-byte Folded Spill 137; FLATSCR-NEXT: scratch_store_dword off, [[CSR_VGPR:v[0-9]+]], s32 ; 4-byte Folded Spill 138; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC0]] 139; MUBUF-DAG: s_add_u32 s32, s32, 0x400 140; FLATSCR-DAG: s_add_u32 s32, s32, 16 141; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s33, [[FP_SPILL_LANE:[0-9]+]] 142 143; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s30, 0 144; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s31, 1 145; GCN: s_swappc_b64 146 147; MUBUF-DAG: v_readlane_b32 s4, v40, 0 148; MUBUF-DAG: v_readlane_b32 s5, v40, 1 149; FLATSCR-DAG: v_readlane_b32 s0, v40, 0 150; FLATSCR-DAG: v_readlane_b32 s1, v40, 1 151 152; MUBUF: s_sub_u32 s32, s32, 0x400 153; FLATSCR: s_sub_u32 s32, s32, 16 154; GCN-NEXT: v_readlane_b32 s33, [[CSR_VGPR]], [[FP_SPILL_LANE]] 155; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}} 156; MUBUF-NEXT: buffer_load_dword [[CSR_VGPR]], off, s[0:3], s32 ; 4-byte Folded Reload 157; FLATSCR-NEXT: scratch_load_dword [[CSR_VGPR]], off, s32 ; 4-byte Folded Reload 158; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]] 159; GCN-NEXT: s_waitcnt vmcnt(0) 160; GCN-NEXT: s_setpc_b64 161define void @callee_no_stack_with_call() #0 { 162 call void @external_void_func_void() 163 ret void 164} 165 166declare hidden void @external_void_func_void() #0 167 168; Make sure if a CSR vgpr is used for SGPR spilling, it is saved and 169; restored. No FP is required. 170; 171; GCN-LABEL: {{^}}callee_func_sgpr_spill_no_calls: 172; GCN: s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}} 173; MUBUF-NEXT: buffer_store_dword [[CSR_VGPR:v[0-9]+]], off, s[0:3], s32 ; 4-byte Folded Spill 174; FLATSCR-NEXT: scratch_store_dword off, [[CSR_VGPR:v[0-9]+]], s32 ; 4-byte Folded Spill 175; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC0]] 176; GCN: v_writelane_b32 [[CSR_VGPR]], s 177; GCN: v_writelane_b32 [[CSR_VGPR]], s 178 179; GCN: ;;#ASMSTART 180; GCN: v_readlane_b32 s{{[0-9]+}}, [[CSR_VGPR]] 181; GCN: v_readlane_b32 s{{[0-9]+}}, [[CSR_VGPR]] 182 183; GCN: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}} 184; MUBUF-NEXT: buffer_load_dword [[CSR_VGPR]], off, s[0:3], s32 ; 4-byte Folded Reload 185; FLATSCR-NEXT: scratch_load_dword [[CSR_VGPR]], off, s32 ; 4-byte Folded Reload 186; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]] 187; GCN-NEXT: s_waitcnt 188; GCN-NEXT: s_setpc_b64 189define void @callee_func_sgpr_spill_no_calls(i32 %in) #0 { 190 call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7}"() #0 191 call void asm sideeffect "", "~{v8},~{v9},~{v10},~{v11},~{v12},~{v13},~{v14},~{v15}"() #0 192 call void asm sideeffect "", "~{v16},~{v17},~{v18},~{v19},~{v20},~{v21},~{v22},~{v23}"() #0 193 call void asm sideeffect "", "~{v24},~{v25},~{v26},~{v27},~{v28},~{v29},~{v30},~{v31}"() #0 194 call void asm sideeffect "", "~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38},~{v39}"() #0 195 196 %wide.sgpr0 = call <16 x i32> asm sideeffect "; def $0", "=s" () #0 197 %wide.sgpr1 = call <16 x i32> asm sideeffect "; def $0", "=s" () #0 198 %wide.sgpr2 = call <16 x i32> asm sideeffect "; def $0", "=s" () #0 199 %wide.sgpr5 = call <16 x i32> asm sideeffect "; def $0", "=s" () #0 200 %wide.sgpr3 = call <8 x i32> asm sideeffect "; def $0", "=s" () #0 201 %wide.sgpr4 = call <2 x i32> asm sideeffect "; def $0", "=s" () #0 202 203 call void asm sideeffect "; use $0", "s"(<16 x i32> %wide.sgpr0) #0 204 call void asm sideeffect "; use $0", "s"(<16 x i32> %wide.sgpr1) #0 205 call void asm sideeffect "; use $0", "s"(<16 x i32> %wide.sgpr2) #0 206 call void asm sideeffect "; use $0", "s"(<8 x i32> %wide.sgpr3) #0 207 call void asm sideeffect "; use $0", "s"(<2 x i32> %wide.sgpr4) #0 208 call void asm sideeffect "; use $0", "s"(<16 x i32> %wide.sgpr5) #0 209 ret void 210} 211 212; Has no spilled CSR VGPRs used for SGPR spilling, so no need to 213; enable all lanes and restore. 214 215; GCN-LABEL: {{^}}spill_only_csr_sgpr: 216; GCN: s_waitcnt 217; GCN-NEXT: s_or_saveexec_b64 218; MUBUF-NEXT: buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill 219; FLATSCR-NEXT: scratch_store_dword off, v0, s32 ; 4-byte Folded Spill 220; GCN-NEXT: s_mov_b64 exec, 221; GCN-NEXT: v_writelane_b32 v0, s42, 0 222; GCN-NEXT: ;;#ASMSTART 223; GCN-NEXT: ; clobber s42 224; GCN-NEXT: ;;#ASMEND 225; GCN-NEXT: v_readlane_b32 s42, v0, 0 226; GCN-NEXT: s_or_saveexec_b64 227; MUBUF-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload 228; FLATSCR-NEXT: scratch_load_dword v0, off, s32 ; 4-byte Folded Reload 229; GCN-NEXT: s_mov_b64 exec, 230; GCN-NEXT: s_waitcnt vmcnt(0) 231; GCN-NEXT: s_setpc_b64 232define void @spill_only_csr_sgpr() { 233 call void asm sideeffect "; clobber s42", "~{s42}"() 234 ret void 235} 236 237; TODO: Can the SP inc/deec be remvoed? 238; GCN-LABEL: {{^}}callee_with_stack_no_fp_elim_csr_vgpr: 239; GCN: s_waitcnt 240; GCN-NEXT:s_mov_b32 [[FP_COPY:s[0-9]+]], s33 241; GCN-NEXT: s_mov_b32 s33, s32 242; GCN: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0 243; MUBUF-DAG: buffer_store_dword v41, off, s[0:3], s33 ; 4-byte Folded Spill 244; FLATSCR-DAG: scratch_store_dword off, v41, s33 ; 4-byte Folded Spill 245; MUBUF-DAG: buffer_store_dword [[ZERO]], off, s[0:3], s33 offset:8 246; FLATSCR-DAG: scratch_store_dword off, [[ZERO]], s33 offset:8 247 248; GCN: ;;#ASMSTART 249; GCN-NEXT: ; clobber v41 250; GCN-NEXT: ;;#ASMEND 251 252; MUBUF: buffer_load_dword v41, off, s[0:3], s33 ; 4-byte Folded Reload 253; FLATSCR: scratch_load_dword v41, off, s33 ; 4-byte Folded Reload 254; MUBUF: s_add_u32 s32, s32, 0x300 255; MUBUF-NEXT: s_sub_u32 s32, s32, 0x300 256; MUBUF-NEXT: s_mov_b32 s33, s4 257; FLATSCR: s_add_u32 s32, s32, 12 258; FLATSCR-NEXT: s_sub_u32 s32, s32, 12 259; FLATSCR-NEXT: s_mov_b32 s33, s0 260; GCN-NEXT: s_waitcnt vmcnt(0) 261; GCN-NEXT: s_setpc_b64 262define void @callee_with_stack_no_fp_elim_csr_vgpr() #1 { 263 %alloca = alloca i32, addrspace(5) 264 store volatile i32 0, i32 addrspace(5)* %alloca 265 call void asm sideeffect "; clobber v41", "~{v41}"() 266 ret void 267} 268 269; Use a copy to a free SGPR instead of introducing a second CSR VGPR. 270; GCN-LABEL: {{^}}last_lane_vgpr_for_fp_csr: 271; GCN: s_waitcnt 272; GCN-NEXT: v_writelane_b32 v1, s33, 63 273; GCN-COUNT-60: v_writelane_b32 v1 274; GCN: s_mov_b32 s33, s32 275; GCN-COUNT-2: v_writelane_b32 v1 276; MUBUF: buffer_store_dword v41, off, s[0:3], s33 ; 4-byte Folded Spill 277; FLATSCR: scratch_store_dword off, v41, s33 ; 4-byte Folded Spill 278; MUBUF: buffer_store_dword v{{[0-9]+}}, off, s[0:3], s33 offset:8 279; FLATSCR: scratch_store_dword off, v{{[0-9]+}}, s33 offset:8 280; GCN: ;;#ASMSTART 281; GCN: v_writelane_b32 v1 282 283; MUBUF: s_add_u32 s32, s32, 0x300 284; MUBUF: s_sub_u32 s32, s32, 0x300 285; FLATSCR: s_add_u32 s32, s32, 12 286; FLATSCR: s_sub_u32 s32, s32, 12 287; GCN-NEXT: v_readlane_b32 s33, v1, 63 288; GCN-NEXT: s_waitcnt vmcnt(0) 289; GCN-NEXT: s_setpc_b64 290define void @last_lane_vgpr_for_fp_csr() #1 { 291 %alloca = alloca i32, addrspace(5) 292 store volatile i32 0, i32 addrspace(5)* %alloca 293 call void asm sideeffect "; clobber v41", "~{v41}"() 294 call void asm sideeffect "", 295 "~{s40},~{s41},~{s42},~{s43},~{s44},~{s45},~{s46},~{s47},~{s48},~{s49} 296 ,~{s50},~{s51},~{s52},~{s53},~{s54},~{s55},~{s56},~{s57},~{s58},~{s59} 297 ,~{s60},~{s61},~{s62},~{s63},~{s64},~{s65},~{s66},~{s67},~{s68},~{s69} 298 ,~{s70},~{s71},~{s72},~{s73},~{s74},~{s75},~{s76},~{s77},~{s78},~{s79} 299 ,~{s80},~{s81},~{s82},~{s83},~{s84},~{s85},~{s86},~{s87},~{s88},~{s89} 300 ,~{s90},~{s91},~{s92},~{s93},~{s94},~{s95},~{s96},~{s97},~{s98},~{s99} 301 ,~{s100},~{s101},~{s102}"() #1 302 303 ret void 304} 305 306; Use a copy to a free SGPR instead of introducing a second CSR VGPR. 307; GCN-LABEL: {{^}}no_new_vgpr_for_fp_csr: 308; GCN: s_waitcnt 309; GCN-COUNT-62: v_writelane_b32 v1, 310; GCN: s_mov_b32 [[FP_COPY:s[0-9]+]], s33 311; GCN-NEXT: s_mov_b32 s33, s32 312; GCN: v_writelane_b32 v1, 313; MUBUF: buffer_store_dword v41, off, s[0:3], s33 ; 4-byte Folded Spill 314; FLATSCR: scratch_store_dword off, v41, s33 ; 4-byte Folded Spill 315; MUBUF: buffer_store_dword 316; FLATSCR: scratch_store_dword 317; GCN: ;;#ASMSTART 318; GCN: v_writelane_b32 v1, 319; MUBUF: buffer_load_dword v41, off, s[0:3], s33 ; 4-byte Folded Reload 320; FLATSCR: scratch_load_dword v41, off, s33 ; 4-byte Folded Reload 321; MUBUF: s_add_u32 s32, s32, 0x300 322; FLATSCR: s_add_u32 s32, s32, 12 323; GCN-COUNT-64: v_readlane_b32 s{{[0-9]+}}, v1 324; MUBUF-NEXT: s_sub_u32 s32, s32, 0x300 325; FLATSCR-NEXT: s_sub_u32 s32, s32, 12 326; GCN-NEXT: s_mov_b32 s33, [[FP_COPY]] 327; GCN-NEXT: s_waitcnt vmcnt(0) 328; GCN-NEXT: s_setpc_b64 329define void @no_new_vgpr_for_fp_csr() #1 { 330 %alloca = alloca i32, addrspace(5) 331 store volatile i32 0, i32 addrspace(5)* %alloca 332 call void asm sideeffect "; clobber v41", "~{v41}"() 333 call void asm sideeffect "", 334 "~{s39},~{s40},~{s41},~{s42},~{s43},~{s44},~{s45},~{s46},~{s47},~{s48},~{s49} 335 ,~{s50},~{s51},~{s52},~{s53},~{s54},~{s55},~{s56},~{s57},~{s58},~{s59} 336 ,~{s60},~{s61},~{s62},~{s63},~{s64},~{s65},~{s66},~{s67},~{s68},~{s69} 337 ,~{s70},~{s71},~{s72},~{s73},~{s74},~{s75},~{s76},~{s77},~{s78},~{s79} 338 ,~{s80},~{s81},~{s82},~{s83},~{s84},~{s85},~{s86},~{s87},~{s88},~{s89} 339 ,~{s90},~{s91},~{s92},~{s93},~{s94},~{s95},~{s96},~{s97},~{s98},~{s99} 340 ,~{s100},~{s101},~{s102}"() #1 341 342 ret void 343} 344 345; GCN-LABEL: {{^}}realign_stack_no_fp_elim: 346; GCN: s_waitcnt 347; MUBUF-NEXT: s_mov_b32 [[FP_COPY:s4]], s33 348; FLATSCR-NEXT: s_mov_b32 [[FP_COPY:s0]], s33 349; MUBUF-NEXT: s_add_u32 s33, s32, 0x7ffc0 350; FLATSCR-NEXT: s_add_u32 s33, s32, 0x1fff 351; MUBUF-NEXT: s_and_b32 s33, s33, 0xfff80000 352; FLATSCR-NEXT: s_and_b32 s33, s33, 0xffffe000 353; MUBUF-NEXT: s_add_u32 s32, s32, 0x100000 354; FLATSCR-NEXT: s_add_u32 s32, s32, 0x4000 355; GCN-NEXT: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0 356; MUBUF-NEXT: buffer_store_dword [[ZERO]], off, s[0:3], s33 357; FLATSCR-NEXT: scratch_store_dword off, [[ZERO]], s33 358; GCN-NEXT: s_waitcnt vmcnt(0) 359; MUBUF-NEXT: s_sub_u32 s32, s32, 0x100000 360; FLATSCR-NEXT: s_sub_u32 s32, s32, 0x4000 361; GCN-NEXT: s_mov_b32 s33, [[FP_COPY]] 362; GCN-NEXT: s_setpc_b64 363define void @realign_stack_no_fp_elim() #1 { 364 %alloca = alloca i32, align 8192, addrspace(5) 365 store volatile i32 0, i32 addrspace(5)* %alloca 366 ret void 367} 368 369; GCN-LABEL: {{^}}no_unused_non_csr_sgpr_for_fp: 370; GCN: s_waitcnt 371; GCN-NEXT: v_writelane_b32 v1, s33, 2 372; GCN-NEXT: v_writelane_b32 v1, s30, 0 373; GCN-NEXT: s_mov_b32 s33, s32 374; GCN: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0 375; GCN: v_writelane_b32 v1, s31, 1 376; MUBUF: buffer_store_dword [[ZERO]], off, s[0:3], s33 offset:4 377; FLATSCR: scratch_store_dword off, [[ZERO]], s33 offset:4 378; GCN-NEXT: s_waitcnt vmcnt(0) 379; GCN: ;;#ASMSTART 380; MUBUF: v_readlane_b32 s4, v1, 0 381; MUBUF-NEXT: s_add_u32 s32, s32, 0x200 382; MUBUF-NEXT: v_readlane_b32 s5, v1, 1 383; FLATSCR: v_readlane_b32 s0, v1, 0 384; FLATSCR-NEXT: s_add_u32 s32, s32, 8 385; FLATSCR-NEXT: v_readlane_b32 s1, v1, 1 386; MUBUF-NEXT: s_sub_u32 s32, s32, 0x200 387; FLATSCR-NEXT: s_sub_u32 s32, s32, 8 388; GCN-NEXT: v_readlane_b32 s33, v1, 2 389; MUBUF-NEXT: s_setpc_b64 s[4:5] 390; FLATSCR-NEXT: s_setpc_b64 s[0:1] 391define void @no_unused_non_csr_sgpr_for_fp() #1 { 392 %alloca = alloca i32, addrspace(5) 393 store volatile i32 0, i32 addrspace(5)* %alloca 394 395 ; Use all clobberable registers, so FP has to spill to a VGPR. 396 call void asm sideeffect "", 397 "~{s0},~{s1},~{s2},~{s3},~{s4},~{s5},~{s6},~{s7},~{s8},~{s9} 398 ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19} 399 ,~{s20},~{s21},~{s22},~{s23},~{s24},~{s25},~{s26},~{s27},~{s28},~{s29} 400 ,~{s30},~{s31}"() #0 401 402 ret void 403} 404 405; Need a new CSR VGPR to satisfy the FP spill. 406; GCN-LABEL: {{^}}no_unused_non_csr_sgpr_for_fp_no_scratch_vgpr: 407; GCN: s_waitcnt 408; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}} 409; MUBUF-NEXT: buffer_store_dword [[CSR_VGPR:v[0-9]+]], off, s[0:3], s32 offset:8 ; 4-byte Folded Spill 410; FLATSCR-NEXT: scratch_store_dword off, [[CSR_VGPR:v[0-9]+]], s32 offset:8 ; 4-byte Folded Spill 411; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC0]] 412; GCN-NEXT: v_writelane_b32 [[CSR_VGPR]], s33, 2 413; GCN-NEXT: v_writelane_b32 [[CSR_VGPR]], s30, 0 414; GCN-NEXT: s_mov_b32 s33, s32 415 416; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s31, 1 417; MUBUF-DAG: buffer_store_dword 418; FLATSCR-DAG: scratch_store_dword 419; MUBUF: s_add_u32 s32, s32, 0x300{{$}} 420; FLATSCR: s_add_u32 s32, s32, 12{{$}} 421 422; MUBUF: v_readlane_b32 s4, [[CSR_VGPR]], 0 423; FLATSCR: v_readlane_b32 s0, [[CSR_VGPR]], 0 424; GCN: ;;#ASMSTART 425; MUBUF: v_readlane_b32 s5, [[CSR_VGPR]], 1 426; FLATSCR: v_readlane_b32 s1, [[CSR_VGPR]], 1 427; MUBUF-NEXT: s_sub_u32 s32, s32, 0x300{{$}} 428; FLATSCR-NEXT: s_sub_u32 s32, s32, 12{{$}} 429; GCN-NEXT: v_readlane_b32 s33, [[CSR_VGPR]], 2 430; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}} 431; MUBUF-NEXT: buffer_load_dword [[CSR_VGPR]], off, s[0:3], s32 offset:8 ; 4-byte Folded Reload 432; FLATSCR-NEXT: scratch_load_dword [[CSR_VGPR]], off, s32 offset:8 ; 4-byte Folded Reload 433; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]] 434; GCN-NEXT: s_waitcnt vmcnt(0) 435; GCN-NEXT: s_setpc_b64 436define void @no_unused_non_csr_sgpr_for_fp_no_scratch_vgpr() #1 { 437 %alloca = alloca i32, addrspace(5) 438 store volatile i32 0, i32 addrspace(5)* %alloca 439 440 ; Use all clobberable registers, so FP has to spill to a VGPR. 441 call void asm sideeffect "", 442 "~{s0},~{s1},~{s2},~{s3},~{s4},~{s5},~{s6},~{s7},~{s8},~{s9} 443 ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19} 444 ,~{s20},~{s21},~{s22},~{s23},~{s24},~{s25},~{s26},~{s27},~{s28},~{s29} 445 ,~{s30},~{s31}"() #0 446 447 call void asm sideeffect "; clobber nonpreserved initial VGPRs", 448 "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9} 449 ,~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19} 450 ,~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29} 451 ,~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38},~{v39}"() #1 452 453 ret void 454} 455 456; The byval argument exceeds the MUBUF constant offset, so a scratch 457; register is needed to access the CSR VGPR slot. 458; GCN-LABEL: {{^}}scratch_reg_needed_mubuf_offset: 459; GCN: s_waitcnt 460; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}} 461; MUBUF-NEXT: v_mov_b32_e32 [[SCRATCH_VGPR:v[0-9]+]], 0x1008 462; MUBUF-NEXT: buffer_store_dword [[CSR_VGPR:v[0-9]+]], [[SCRATCH_VGPR]], s[0:3], s32 offen ; 4-byte Folded Spill 463; FLATSCR-NEXT: s_add_u32 [[SCRATCH_SGPR:s[0-9]+]], s32, 0x1008 464; FLATSCR-NEXT: scratch_store_dword off, [[CSR_VGPR:v[0-9]+]], [[SCRATCH_SGPR]] ; 4-byte Folded Spill 465; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC0]] 466; GCN-NEXT: v_writelane_b32 [[CSR_VGPR]], s33, 2 467; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s30, 0 468; GCN-DAG: s_mov_b32 s33, s32 469; GCN-DAG: v_writelane_b32 [[CSR_VGPR]], s31, 1 470; MUBUF-DAG: s_add_u32 s32, s32, 0x40300{{$}} 471; FLATSCR-DAG: s_add_u32 s32, s32, 0x100c{{$}} 472; MUBUF-DAG: buffer_store_dword 473; FLATSCR-DAG: scratch_store_dword 474 475; MUBUF: v_readlane_b32 s4, [[CSR_VGPR]], 0 476; FLATSCR: v_readlane_b32 s0, [[CSR_VGPR]], 0 477; GCN: ;;#ASMSTART 478; MUBUF: v_readlane_b32 s5, [[CSR_VGPR]], 1 479; FLATSCR: v_readlane_b32 s1, [[CSR_VGPR]], 1 480; MUBUF-NEXT: s_sub_u32 s32, s32, 0x40300{{$}} 481; FLATSCR-NEXT: s_sub_u32 s32, s32, 0x100c{{$}} 482; GCN-NEXT: v_readlane_b32 s33, [[CSR_VGPR]], 2 483; GCN-NEXT: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}} 484; MUBUF-NEXT: v_mov_b32_e32 [[SCRATCH_VGPR:v[0-9]+]], 0x1008 485; MUBUF-NEXT: buffer_load_dword [[CSR_VGPR]], [[SCRATCH_VGPR]], s[0:3], s32 offen ; 4-byte Folded Reload 486; FLATSCR-NEXT: s_add_u32 [[SCRATCH_SGPR:s[0-9]+]], s32, 0x1008 487; FLATSCR-NEXT: scratch_load_dword [[CSR_VGPR]], off, [[SCRATCH_SGPR]] ; 4-byte Folded Reload 488; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]] 489; GCN-NEXT: s_waitcnt vmcnt(0) 490; GCN-NEXT: s_setpc_b64 491define void @scratch_reg_needed_mubuf_offset([4096 x i8] addrspace(5)* byval([4096 x i8]) align 4 %arg) #1 { 492 %alloca = alloca i32, addrspace(5) 493 store volatile i32 0, i32 addrspace(5)* %alloca 494 495 ; Use all clobberable registers, so FP has to spill to a VGPR. 496 call void asm sideeffect "; clobber nonpreserved SGPRs", 497 "~{s0},~{s1},~{s2},~{s3},~{s4},~{s5},~{s6},~{s7},~{s8},~{s9} 498 ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19} 499 ,~{s20},~{s21},~{s22},~{s23},~{s24},~{s25},~{s26},~{s27},~{s28},~{s29} 500 ,~{s30},~{s31}"() #0 501 502 ; Use all clobberable VGPRs, so a CSR spill is needed for the VGPR 503 call void asm sideeffect "; clobber nonpreserved VGPRs", 504 "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9} 505 ,~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19} 506 ,~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29} 507 ,~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38},~{v39}"() #1 508 509 ret void 510} 511 512; GCN-LABEL: {{^}}local_empty_func: 513; GCN: s_waitcnt 514; GCN-NEXT: s_setpc_b64 515define internal void @local_empty_func() #0 { 516 ret void 517} 518 519; An FP is needed, despite not needing any spills 520; TODO: Ccould see callee does not use stack and omit FP. 521; GCN-LABEL: {{^}}ipra_call_with_stack: 522; GCN: s_mov_b32 [[FP_COPY:s[0-9]+]], s33 523; GCN: s_mov_b32 s33, s32 524; MUBUF: s_add_u32 s32, s32, 0x400 525; FLATSCR: s_add_u32 s32, s32, 16 526; MUBUF: buffer_store_dword v{{[0-9]+}}, off, s[0:3], s33{{$}} 527; FLATSCR: scratch_store_dword off, v{{[0-9]+}}, s33{{$}} 528; GCN: s_swappc_b64 529; MUBUF: s_sub_u32 s32, s32, 0x400 530; FLATSCR: s_sub_u32 s32, s32, 16 531; GCN: s_mov_b32 s33, [[FP_COPY:s[0-9]+]] 532define void @ipra_call_with_stack() #0 { 533 %alloca = alloca i32, addrspace(5) 534 store volatile i32 0, i32 addrspace(5)* %alloca 535 call void @local_empty_func() 536 ret void 537} 538 539; With no free registers, we must spill the FP to memory. 540; GCN-LABEL: {{^}}callee_need_to_spill_fp_to_memory: 541; MUBUF: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}} 542; MUBUF: v_mov_b32_e32 [[TMP_VGPR1:v[0-9]+]], s33 543; MUBUF: buffer_store_dword [[TMP_VGPR1]], off, s[0:3], s32 offset:4 544; MUBUF: s_mov_b64 exec, [[COPY_EXEC1]] 545; FLATSCR: s_mov_b32 s0, s33 546; GCN: s_mov_b32 s33, s32 547; MUBUF: s_or_saveexec_b64 [[COPY_EXEC2:s\[[0-9]+:[0-9]+\]]], -1{{$}} 548; MUBUF: buffer_load_dword [[TMP_VGPR2:v[0-9]+]], off, s[0:3], s32 offset:4 549; FLATSCR: s_mov_b32 s33, s0 550; MUBUF: s_waitcnt vmcnt(0) 551; MUBUF: v_readfirstlane_b32 s33, [[TMP_VGPR2]] 552; MUBUF: s_mov_b64 exec, [[COPY_EXEC2]] 553; GCN: s_setpc_b64 554; MUBUF: ScratchSize: 8 555; FLATSCR: ScratchSize: 0 556define void @callee_need_to_spill_fp_to_memory() #3 { 557 call void asm sideeffect "; clobber nonpreserved SGPRs", 558 "~{s4},~{s5},~{s6},~{s7},~{s8},~{s9} 559 ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19} 560 ,~{s20},~{s21},~{s22},~{s23},~{s24},~{s25},~{s26},~{s27},~{s28},~{s29} 561 ,~{vcc}"() 562 563 call void asm sideeffect "; clobber all VGPRs", 564 "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9} 565 ,~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19} 566 ,~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29} 567 ,~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38},~{v39}"() 568 ret void 569} 570 571; If we have a reserved VGPR that can be used for SGPR spills, we may still 572; need to spill the FP to memory if there are no free lanes in the reserved 573; VGPR. 574; GCN-LABEL: {{^}}callee_need_to_spill_fp_to_memory_full_reserved_vgpr: 575; MUBUF: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}} 576; MUBUF: v_mov_b32_e32 [[TMP_VGPR1:v[0-9]+]], s33 577; MUBUF: buffer_store_dword [[TMP_VGPR1]], off, s[0:3], s32 offset:[[OFF:[0-9]+]] 578; MUBUF: s_mov_b64 exec, [[COPY_EXEC1]] 579; GCN-NOT: v_writelane_b32 v40, s33 580; MUBUF: s_mov_b32 s33, s32 581; FLATSCR: s_mov_b32 s33, s0 582; GCN-NOT: v_readlane_b32 s33, v40 583; MUBUF: s_or_saveexec_b64 [[COPY_EXEC2:s\[[0-9]+:[0-9]+\]]], -1{{$}} 584; MUBUF: buffer_load_dword [[TMP_VGPR2:v[0-9]+]], off, s[0:3], s32 offset:[[OFF]] 585; MUBUF: v_readfirstlane_b32 s33, [[TMP_VGPR2]] 586; MUBUF: s_mov_b64 exec, [[COPY_EXEC2]] 587; GCN: s_setpc_b64 588define void @callee_need_to_spill_fp_to_memory_full_reserved_vgpr() #3 { 589 call void asm sideeffect "; clobber nonpreserved SGPRs and 64 CSRs", 590 "~{s4},~{s5},~{s6},~{s7},~{s8},~{s9} 591 ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19} 592 ,~{s20},~{s21},~{s22},~{s23},~{s24},~{s25},~{s26},~{s27},~{s28},~{s29} 593 ,~{s40},~{s41},~{s42},~{s43},~{s44},~{s45},~{s46},~{s47},~{s48},~{s49} 594 ,~{s50},~{s51},~{s52},~{s53},~{s54},~{s55},~{s56},~{s57},~{s58},~{s59} 595 ,~{s60},~{s61},~{s62},~{s63},~{s64},~{s65},~{s66},~{s67},~{s68},~{s69} 596 ,~{s70},~{s71},~{s72},~{s73},~{s74},~{s75},~{s76},~{s77},~{s78},~{s79} 597 ,~{s80},~{s81},~{s82},~{s83},~{s84},~{s85},~{s86},~{s87},~{s88},~{s89} 598 ,~{s90},~{s91},~{s92},~{s93},~{s94},~{s95},~{s96},~{s97},~{s98},~{s99} 599 ,~{s100},~{s101},~{s102},~{s39},~{vcc}"() 600 601 call void asm sideeffect "; clobber all VGPRs except CSR v40", 602 "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9} 603 ,~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19} 604 ,~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29} 605 ,~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38}"() 606 ret void 607} 608 609; When flat-scratch is enabled, we save the FP to s0. At the same time, 610; the exec register is saved to s0 when saving CSR in the function prolog. 611; Make sure that the FP save happens after restoring exec from the same 612; register. 613; GCN-LABEL: {{^}}callee_need_to_spill_fp_to_reg: 614; GCN-NOT: v_writelane_b32 v40, s33 615; FLATSCR: s_or_saveexec_b64 s[0:1], -1 616; FLATSCR: s_mov_b64 exec, s[0:1] 617; FLATSCR: s_mov_b32 s0, s33 618; FLATSCR: s_mov_b32 s33, s32 619; FLATSCR: s_mov_b32 s33, s0 620; FLATSCR: s_or_saveexec_b64 s[0:1], -1 621; GCN-NOT: v_readlane_b32 s33, v40 622; GCN: s_setpc_b64 623define void @callee_need_to_spill_fp_to_reg() #1 { 624 call void asm sideeffect "; clobber nonpreserved SGPRs and 64 CSRs", 625 "~{s4},~{s5},~{s6},~{s7},~{s8},~{s9} 626 ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19} 627 ,~{s20},~{s21},~{s22},~{s23},~{s24},~{s25},~{s26},~{s27},~{s28},~{s29} 628 ,~{s40},~{s41},~{s42},~{s43},~{s44},~{s45},~{s46},~{s47},~{s48},~{s49} 629 ,~{s50},~{s51},~{s52},~{s53},~{s54},~{s55},~{s56},~{s57},~{s58},~{s59} 630 ,~{s60},~{s61},~{s62},~{s63},~{s64},~{s65},~{s66},~{s67},~{s68},~{s69} 631 ,~{s70},~{s71},~{s72},~{s73},~{s74},~{s75},~{s76},~{s77},~{s78},~{s79} 632 ,~{s80},~{s81},~{s82},~{s83},~{s84},~{s85},~{s86},~{s87},~{s88},~{s89} 633 ,~{s90},~{s91},~{s92},~{s93},~{s94},~{s95},~{s96},~{s97},~{s98},~{s99} 634 ,~{s100},~{s101},~{s102},~{s39},~{vcc}"() 635 636 call void asm sideeffect "; clobber all VGPRs except CSR v40", 637 "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9} 638 ,~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19} 639 ,~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29} 640 ,~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38},~{v39}"() 641 ret void 642} 643 644; If the size of the offset exceeds the MUBUF offset field we need another 645; scratch VGPR to hold the offset. 646; GCN-LABEL: {{^}}spill_fp_to_memory_scratch_reg_needed_mubuf_offset 647; MUBUF: s_or_saveexec_b64 s[4:5], -1 648; MUBUF: v_mov_b32_e32 v0, s33 649; GCN-NOT: v_mov_b32_e32 v0, 0x1008 650; MUBUF-NEXT: v_mov_b32_e32 v1, 0x1008 651; MUBUF-NEXT: buffer_store_dword v0, v1, s[0:3], s32 offen ; 4-byte Folded Spill 652; FLATSCR: s_add_u32 [[SOFF:s[0-9]+]], s33, 0x1004 653; FLATSCR: v_mov_b32_e32 v0, 0 654; FLATSCR: scratch_store_dword off, v0, [[SOFF]] 655define void @spill_fp_to_memory_scratch_reg_needed_mubuf_offset([4096 x i8] addrspace(5)* byval([4096 x i8]) align 4 %arg) #3 { 656 %alloca = alloca i32, addrspace(5) 657 store volatile i32 0, i32 addrspace(5)* %alloca 658 659 call void asm sideeffect "; clobber nonpreserved SGPRs and 64 CSRs", 660 "~{s4},~{s5},~{s6},~{s7},~{s8},~{s9} 661 ,~{s10},~{s11},~{s12},~{s13},~{s14},~{s15},~{s16},~{s17},~{s18},~{s19} 662 ,~{s20},~{s21},~{s22},~{s23},~{s24},~{s25},~{s26},~{s27},~{s28},~{s29} 663 ,~{s40},~{s41},~{s42},~{s43},~{s44},~{s45},~{s46},~{s47},~{s48},~{s49} 664 ,~{s50},~{s51},~{s52},~{s53},~{s54},~{s55},~{s56},~{s57},~{s58},~{s59} 665 ,~{s60},~{s61},~{s62},~{s63},~{s64},~{s65},~{s66},~{s67},~{s68},~{s69} 666 ,~{s70},~{s71},~{s72},~{s73},~{s74},~{s75},~{s76},~{s77},~{s78},~{s79} 667 ,~{s80},~{s81},~{s82},~{s83},~{s84},~{s85},~{s86},~{s87},~{s88},~{s89} 668 ,~{s90},~{s91},~{s92},~{s93},~{s94},~{s95},~{s96},~{s97},~{s98},~{s99} 669 ,~{s100},~{s101},~{s102},~{s39},~{vcc}"() 670 671 call void asm sideeffect "; clobber all VGPRs except CSR v40", 672 "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9} 673 ,~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19} 674 ,~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29} 675 ,~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38}"() 676 ret void 677} 678 679attributes #0 = { nounwind } 680attributes #1 = { nounwind "frame-pointer"="all" } 681attributes #2 = { nounwind "frame-pointer"="non-leaf" } 682attributes #3 = { nounwind "frame-pointer"="all" "amdgpu-waves-per-eu"="6,6" } 683