1; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,LOOP %s 2; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,LOOP %s 3; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,LOOP %s 4; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,NOLOOP,NOLOOP-SDAG %s 5; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -asm-verbose=0 -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,NOLOOP,NOLOOP-SDAG %s 6; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 -asm-verbose=0 -o - -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,NOLOOP,NOLOOP-SDAG %s 7 8; Minimum offset 9; GCN-LABEL: {{^}}gws_init_offset0: 10; GCN-DAG: s_load_{{dword|b32}} [[BAR_NUM:s[0-9]+]] 11; GCN-DAG: s_mov_b32 m0, 0{{$}} 12; GCN: v_mov_b32_e32 v0, [[BAR_NUM]] 13; NOLOOP: ds_gws_init v0 gds{{$}} 14 15; LOOP: [[LOOP:.LBB[0-9]+_[0-9]+]]: 16; LOOP-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 8, 1), 0 17; LOOP-NEXT: ds_gws_init v0 gds 18; LOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 19; LOOP-NEXT: s_getreg_b32 [[GETREG:s[0-9]+]], hwreg(HW_REG_TRAPSTS, 8, 1) 20; LOOP-NEXT: s_cmp_lg_u32 [[GETREG]], 0 21; LOOP-NEXT: s_cbranch_scc1 [[LOOP]] 22define amdgpu_kernel void @gws_init_offset0(i32 %val) #0 { 23 call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 0) 24 ret void 25} 26 27; Maximum offset 28; GCN-LABEL: {{^}}gws_init_offset63: 29; NOLOOP-DAG: s_load_{{dword|b32}} [[BAR_NUM:s[0-9]+]] 30; NOLOOP-DAG: s_mov_b32 m0, 0{{$}} 31; NOLOOP-DAG: v_mov_b32_e32 v0, [[BAR_NUM]] 32; NOLOOP: ds_gws_init v0 offset:63 gds{{$}} 33 34 35; LOOP: s_mov_b32 m0, 0{{$}} 36; LOOP: [[LOOP:.LBB[0-9]+_[0-9]+]]: 37; LOOP-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 8, 1), 0 38; LOOP-NEXT: ds_gws_init v0 offset:63 gds 39; LOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 40; LOOP-NEXT: s_getreg_b32 [[GETREG:s[0-9]+]], hwreg(HW_REG_TRAPSTS, 8, 1) 41; LOOP-NEXT: s_cmp_lg_u32 [[GETREG]], 0 42; LOOP-NEXT: s_cbranch_scc1 [[LOOP]] 43define amdgpu_kernel void @gws_init_offset63(i32 %val) #0 { 44 call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 63) 45 ret void 46} 47 48; FIXME: Should be able to shift directly into m0 49; GCN-LABEL: {{^}}gws_init_sgpr_offset: 50; NOLOOP-DAG: s_load_{{dwordx2|b64}} s[[[BAR_NUM:[0-9]+]]:[[OFFSET:[0-9]+]]] 51 52; NOLOOP-SDAG-DAG: s_lshl_b32 [[SHL:s[0-9]+]], s[[OFFSET]], 16 53; NOLOOP-SDAG-DAG: s_mov_b32 m0, [[SHL]]{{$}} 54 55; NOLOOP-GISEL-DAG: s_lshl_b32 m0, s[[OFFSET]], 16 56 57; NOLOOP-DAG: v_mov_b32_e32 [[GWS_VAL:v[0-9]+]], s[[BAR_NUM]] 58; NOLOOP: ds_gws_init [[GWS_VAL]] gds{{$}} 59define amdgpu_kernel void @gws_init_sgpr_offset(i32 %val, i32 %offset) #0 { 60 call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 %offset) 61 ret void 62} 63 64; Variable offset in SGPR with constant add 65; GCN-LABEL: {{^}}gws_init_sgpr_offset_add1: 66; NOLOOP-DAG: s_load_{{dwordx2|b64}} s[[[BAR_NUM:[0-9]+]]:[[OFFSET:[0-9]+]]] 67 68; NOLOOP-SDAG-DAG: s_lshl_b32 [[SHL:s[0-9]+]], s[[OFFSET]], 16 69; NOLOOP-SDAG-DAG: s_mov_b32 m0, [[SHL]]{{$}} 70 71; NOLOOP-GISEL-DAG: s_lshl_b32 m0, s[[OFFSET]], 16 72 73; NOLOOP-DAG: v_mov_b32_e32 [[GWS_VAL:v[0-9]+]], s[[BAR_NUM]] 74; NOLOOP: ds_gws_init [[GWS_VAL]] offset:1 gds{{$}} 75define amdgpu_kernel void @gws_init_sgpr_offset_add1(i32 %val, i32 %offset.base) #0 { 76 %offset = add i32 %offset.base, 1 77 call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 %offset) 78 ret void 79} 80 81; GCN-LABEL: {{^}}gws_init_vgpr_offset: 82; NOLOOP-DAG: s_load_{{dword|b32}} [[BAR_NUM:s[0-9]+]] 83; NOLOOP-DAG: v_readfirstlane_b32 [[READLANE:s[0-9]+]], v0 84 85; NOLOOP-SDAG-DAG: s_lshl_b32 [[SHL:s[0-9]+]], [[READLANE]], 16 86; NOLOOP-SDAG-DAG: s_mov_b32 m0, [[SHL]]{{$}} 87 88; NOLOOP-GISEL-DAG: s_lshl_b32 m0, [[READLANE]], 16 89 90; NOLOOP-DAG: v_mov_b32_e32 v0, [[BAR_NUM]] 91; NOLOOP: ds_gws_init v0 gds{{$}} 92define amdgpu_kernel void @gws_init_vgpr_offset(i32 %val) #0 { 93 %vgpr.offset = call i32 @llvm.amdgcn.workitem.id.x() 94 call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 %vgpr.offset) 95 ret void 96} 97 98; Variable offset in VGPR with constant add 99; GCN-LABEL: {{^}}gws_init_vgpr_offset_add: 100; NOLOOP-DAG: s_load_{{dword|b32}} [[BAR_NUM:s[0-9]+]] 101; NOLOOP-DAG: v_readfirstlane_b32 [[READLANE:s[0-9]+]], v0 102 103; NOLOOP-SDAG-DAG: s_lshl_b32 [[SHL:s[0-9]+]], [[READLANE]], 16 104; NOLOOP-SDAG-DAG: s_mov_b32 m0, [[SHL]]{{$}} 105 106; NOLOOP-GISEL-DAG: s_lshl_b32 m0, [[READLANE]], 16 107 108; NOLOOP-DAG: v_mov_b32_e32 v0, [[BAR_NUM]] 109; NOLOOP: ds_gws_init v0 offset:3 gds{{$}} 110define amdgpu_kernel void @gws_init_vgpr_offset_add(i32 %val) #0 { 111 %vgpr.offset.base = call i32 @llvm.amdgcn.workitem.id.x() 112 %vgpr.offset = add i32 %vgpr.offset.base, 3 113 call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 %vgpr.offset) 114 ret void 115} 116 117@lds = internal unnamed_addr addrspace(3) global i32 undef 118 119; Check if m0 initialization is shared. 120; GCN-LABEL: {{^}}gws_init_save_m0_init_constant_offset: 121; NOLOOP: s_mov_b32 m0, 0 122; NOLOOP: ds_gws_init v{{[0-9]+}} offset:10 gds 123 124; LOOP: s_mov_b32 m0, -1 125; LOOP: ds_write_b32 126; LOOP: s_mov_b32 m0, 0 127; LOOP: s_setreg_imm32_b32 128; LOOP: ds_gws_init v{{[0-9]+}} offset:10 gds 129; LOOP: s_cbranch_scc1 130 131; LOOP: s_mov_b32 m0, -1 132; LOOP: ds_write_b32 133define amdgpu_kernel void @gws_init_save_m0_init_constant_offset(i32 %val) #0 { 134 store volatile i32 1, i32 addrspace(3)* @lds 135 call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 10) 136 store i32 2, i32 addrspace(3)* @lds 137 ret void 138} 139 140; GCN-LABEL: {{^}}gws_init_lgkmcnt: 141; NOLOOP: s_mov_b32 m0, 0{{$}} 142; NOLOOP: ds_gws_init v0 gds{{$}} 143; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 144; NOLOOP-NEXT: s_setpc_b64 145define void @gws_init_lgkmcnt(i32 %val) { 146 call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 0) 147 ret void 148} 149 150; Does not imply memory fence on its own 151; GCN-LABEL: {{^}}gws_init_wait_before: 152; NOLOOP: s_waitcnt lgkmcnt(0) 153; NOLOOP-NOT: s_waitcnt 154; NOLOOP: ds_gws_init 155; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) 156define amdgpu_kernel void @gws_init_wait_before(i32 %val, i32 addrspace(1)* %ptr) #0 { 157 store i32 0, i32 addrspace(1)* %ptr 158 call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 7) 159 ret void 160} 161 162declare void @llvm.amdgcn.ds.gws.init(i32, i32) #1 163declare i32 @llvm.amdgcn.workitem.id.x() #2 164 165attributes #0 = { nounwind } 166attributes #1 = { convergent inaccessiblememonly nounwind writeonly } 167attributes #2 = { nounwind readnone speculatable } 168