1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py 2; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-W64 %s 3; RUN: llc -march=amdgcn -mcpu=gfx1030 -verify-machineinstrs -mattr=+wavefrontsize32,-wavefrontsize64 < %s | FileCheck -check-prefixes=GFX10-W32 %s 4 5; Check that WQM isn't triggered by image load/store intrinsics. 6define amdgpu_ps <4 x float> @test1(<8 x i32> inreg %rsrc, i32 %c) { 7; GFX9-W64-LABEL: test1: 8; GFX9-W64: ; %bb.0: ; %main_body 9; GFX9-W64-NEXT: v_mov_b32_e32 v4, v0 10; GFX9-W64-NEXT: image_load v[0:3], v0, s[0:7] dmask:0xf unorm 11; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 12; GFX9-W64-NEXT: image_store v[0:3], v4, s[0:7] dmask:0xf unorm 13; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 14; GFX9-W64-NEXT: ; return to shader part epilog 15; 16; GFX10-W32-LABEL: test1: 17; GFX10-W32: ; %bb.0: ; %main_body 18; GFX10-W32-NEXT: v_mov_b32_e32 v4, v0 19; GFX10-W32-NEXT: image_load v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm 20; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 21; GFX10-W32-NEXT: image_store v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm 22; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 23; GFX10-W32-NEXT: ; return to shader part epilog 24main_body: 25 %tex = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %c, <8 x i32> %rsrc, i32 0, i32 0) 26 call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> %tex, i32 15, i32 %c, <8 x i32> %rsrc, i32 0, i32 0) 27 ret <4 x float> %tex 28} 29 30; Check that WQM is triggered by code calculating inputs to image samples and is disabled as soon as possible 31define amdgpu_ps <4 x float> @test2(i32 inreg, i32 inreg, i32 inreg, i32 inreg %m0, <8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, <2 x float> %pos) #6 { 32; GFX9-W64-LABEL: test2: 33; GFX9-W64: ; %bb.0: ; %main_body 34; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec 35; GFX9-W64-NEXT: s_wqm_b64 exec, exec 36; GFX9-W64-NEXT: s_mov_b32 m0, s3 37; GFX9-W64-NEXT: s_nop 0 38; GFX9-W64-NEXT: v_interp_p1_f32_e32 v2, v0, attr0.x 39; GFX9-W64-NEXT: v_interp_p1_f32_e32 v3, v0, attr0.y 40; GFX9-W64-NEXT: v_interp_p2_f32_e32 v2, v1, attr0.x 41; GFX9-W64-NEXT: v_interp_p2_f32_e32 v3, v1, attr0.y 42; GFX9-W64-NEXT: s_and_b64 exec, exec, s[0:1] 43; GFX9-W64-NEXT: image_sample v[0:3], v[2:3], s[4:11], s[12:15] dmask:0xf 44; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 45; GFX9-W64-NEXT: ; return to shader part epilog 46; 47; GFX10-W32-LABEL: test2: 48; GFX10-W32: ; %bb.0: ; %main_body 49; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo 50; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 51; GFX10-W32-NEXT: s_mov_b32 m0, s3 52; GFX10-W32-NEXT: v_interp_p1_f32_e32 v2, v0, attr0.x 53; GFX10-W32-NEXT: v_interp_p1_f32_e32 v3, v0, attr0.y 54; GFX10-W32-NEXT: v_interp_p2_f32_e32 v2, v1, attr0.x 55; GFX10-W32-NEXT: v_interp_p2_f32_e32 v3, v1, attr0.y 56; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s0 57; GFX10-W32-NEXT: image_sample v[0:3], v[2:3], s[4:11], s[12:15] dmask:0xf dim:SQ_RSRC_IMG_2D 58; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 59; GFX10-W32-NEXT: ; return to shader part epilog 60main_body: 61 %inst23 = extractelement <2 x float> %pos, i32 0 62 %inst24 = extractelement <2 x float> %pos, i32 1 63 %inst25 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 0, i32 0, i32 %m0) 64 %inst26 = tail call float @llvm.amdgcn.interp.p2(float %inst25, float %inst24, i32 0, i32 0, i32 %m0) 65 %inst28 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 1, i32 0, i32 %m0) 66 %inst29 = tail call float @llvm.amdgcn.interp.p2(float %inst28, float %inst24, i32 1, i32 0, i32 %m0) 67 %tex = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32 15, float %inst26, float %inst29, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 68 ret <4 x float> %tex 69} 70 71; ... but disabled for stores (and, in this simple case, not re-enabled) ... 72define amdgpu_ps <4 x float> @test3(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float %c) { 73; GFX9-W64-LABEL: test3: 74; GFX9-W64: ; %bb.0: ; %main_body 75; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec 76; GFX9-W64-NEXT: s_wqm_b64 exec, exec 77; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 78; GFX9-W64-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf 79; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 80; GFX9-W64-NEXT: buffer_store_dwordx4 v[0:3], v0, s[0:3], 0 idxen 81; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 82; GFX9-W64-NEXT: ; return to shader part epilog 83; 84; GFX10-W32-LABEL: test3: 85; GFX10-W32: ; %bb.0: ; %main_body 86; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo 87; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 88; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 89; GFX10-W32-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D 90; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 91; GFX10-W32-NEXT: buffer_store_dwordx4 v[0:3], v0, s[0:3], 0 idxen 92; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 93; GFX10-W32-NEXT: ; return to shader part epilog 94main_body: 95 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 96 %tex.1 = bitcast <4 x float> %tex to <4 x i32> 97 %tex.2 = extractelement <4 x i32> %tex.1, i32 0 98 99 call void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float> %tex, <4 x i32> undef, i32 %tex.2, i32 0, i32 0, i32 0) 100 101 ret <4 x float> %tex 102} 103 104; ... and disabled for export. 105define amdgpu_ps void @test3x(i32 inreg, i32 inreg, i32 inreg, i32 inreg %m0, <8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, <2 x float> %pos) #6 { 106; GFX9-W64-LABEL: test3x: 107; GFX9-W64: ; %bb.0: ; %main_body 108; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec 109; GFX9-W64-NEXT: s_wqm_b64 exec, exec 110; GFX9-W64-NEXT: s_mov_b32 m0, s3 111; GFX9-W64-NEXT: s_nop 0 112; GFX9-W64-NEXT: v_interp_p1_f32_e32 v2, v0, attr0.x 113; GFX9-W64-NEXT: v_interp_p1_f32_e32 v3, v0, attr0.y 114; GFX9-W64-NEXT: v_interp_p2_f32_e32 v2, v1, attr0.x 115; GFX9-W64-NEXT: v_interp_p2_f32_e32 v3, v1, attr0.y 116; GFX9-W64-NEXT: s_and_b64 exec, exec, s[0:1] 117; GFX9-W64-NEXT: image_sample v[0:3], v[2:3], s[4:11], s[12:15] dmask:0xf 118; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 119; GFX9-W64-NEXT: exp mrt0 v0, v1, v2, v3 done vm 120; GFX9-W64-NEXT: s_endpgm 121; 122; GFX10-W32-LABEL: test3x: 123; GFX10-W32: ; %bb.0: ; %main_body 124; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo 125; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 126; GFX10-W32-NEXT: s_mov_b32 m0, s3 127; GFX10-W32-NEXT: v_interp_p1_f32_e32 v2, v0, attr0.x 128; GFX10-W32-NEXT: v_interp_p1_f32_e32 v3, v0, attr0.y 129; GFX10-W32-NEXT: v_interp_p2_f32_e32 v2, v1, attr0.x 130; GFX10-W32-NEXT: v_interp_p2_f32_e32 v3, v1, attr0.y 131; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s0 132; GFX10-W32-NEXT: image_sample v[0:3], v[2:3], s[4:11], s[12:15] dmask:0xf dim:SQ_RSRC_IMG_2D 133; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 134; GFX10-W32-NEXT: exp mrt0 v0, v1, v2, v3 done vm 135; GFX10-W32-NEXT: s_endpgm 136main_body: 137 %inst23 = extractelement <2 x float> %pos, i32 0 138 %inst24 = extractelement <2 x float> %pos, i32 1 139 %inst25 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 0, i32 0, i32 %m0) 140 %inst26 = tail call float @llvm.amdgcn.interp.p2(float %inst25, float %inst24, i32 0, i32 0, i32 %m0) 141 %inst28 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 1, i32 0, i32 %m0) 142 %inst29 = tail call float @llvm.amdgcn.interp.p2(float %inst28, float %inst24, i32 1, i32 0, i32 %m0) 143 %tex = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32 15, float %inst26, float %inst29, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 144 %tex.0 = extractelement <4 x float> %tex, i32 0 145 %tex.1 = extractelement <4 x float> %tex, i32 1 146 %tex.2 = extractelement <4 x float> %tex, i32 2 147 %tex.3 = extractelement <4 x float> %tex, i32 3 148 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %tex.0, float %tex.1, float %tex.2, float %tex.3, i1 true, i1 true) 149 ret void 150} 151 152; Check that WQM is re-enabled when required. 153define amdgpu_ps <4 x float> @test4(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float addrspace(1)* inreg %ptr, i32 %c, i32 %d, float %data) { 154; GFX9-W64-LABEL: test4: 155; GFX9-W64: ; %bb.0: ; %main_body 156; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec 157; GFX9-W64-NEXT: s_wqm_b64 exec, exec 158; GFX9-W64-NEXT: v_mul_lo_u32 v4, v0, v1 159; GFX9-W64-NEXT: image_sample v0, v4, s[0:7], s[8:11] dmask:0x1 160; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 161; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 162; GFX9-W64-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf 163; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 164; GFX9-W64-NEXT: buffer_store_dwordx4 v[0:3], v4, s[0:3], 0 idxen 165; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 166; GFX9-W64-NEXT: ; return to shader part epilog 167; 168; GFX10-W32-LABEL: test4: 169; GFX10-W32: ; %bb.0: ; %main_body 170; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo 171; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 172; GFX10-W32-NEXT: v_mul_lo_u32 v4, v0, v1 173; GFX10-W32-NEXT: image_sample v0, v4, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 174; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 175; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 176; GFX10-W32-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D 177; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 178; GFX10-W32-NEXT: buffer_store_dwordx4 v[0:3], v4, s[0:3], 0 idxen 179; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 180; GFX10-W32-NEXT: ; return to shader part epilog 181main_body: 182 %c.1 = mul i32 %c, %d 183 184 call void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float> undef, <4 x i32> undef, i32 %c.1, i32 0, i32 0, i32 0) 185 %c.1.bc = bitcast i32 %c.1 to float 186 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.1.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 187 %tex0 = extractelement <4 x float> %tex, i32 0 188 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 189 ret <4 x float> %dtex 190} 191 192; Check that WQM is triggered by the wqm intrinsic. 193; WQM was inserting an unecessary v_mov to self after the v_add. Make sure this 194; does not happen - the v_add should write the return reg directly. 195define amdgpu_ps float @test5(i32 inreg %idx0, i32 inreg %idx1) { 196; GFX9-W64-LABEL: test5: 197; GFX9-W64: ; %bb.0: ; %main_body 198; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec 199; GFX9-W64-NEXT: s_wqm_b64 exec, exec 200; GFX9-W64-NEXT: v_mov_b32_e32 v0, s0 201; GFX9-W64-NEXT: v_mov_b32_e32 v1, s1 202; GFX9-W64-NEXT: buffer_load_dword v0, v0, s[0:3], 0 idxen 203; GFX9-W64-NEXT: s_nop 0 204; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 205; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 206; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v1 207; GFX9-W64-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec killed $exec 208; GFX9-W64-NEXT: s_and_b64 exec, exec, s[2:3] 209; GFX9-W64-NEXT: ; return to shader part epilog 210; 211; GFX10-W32-LABEL: test5: 212; GFX10-W32: ; %bb.0: ; %main_body 213; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo 214; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 215; GFX10-W32-NEXT: v_mov_b32_e32 v0, s0 216; GFX10-W32-NEXT: v_mov_b32_e32 v1, s1 217; GFX10-W32-NEXT: s_clause 0x1 218; GFX10-W32-NEXT: buffer_load_dword v0, v0, s[0:3], 0 idxen 219; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 220; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 221; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v1 222; GFX10-W32-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec killed $exec 223; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s2 224; GFX10-W32-NEXT: ; return to shader part epilog 225main_body: 226 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 227 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 228 %out = fadd float %src0, %src1 229 %out.0 = call float @llvm.amdgcn.wqm.f32(float %out) 230 ret float %out.0 231} 232 233; Check that the wqm intrinsic works correctly for integers. 234define amdgpu_ps float @test6(i32 inreg %idx0, i32 inreg %idx1) { 235; GFX9-W64-LABEL: test6: 236; GFX9-W64: ; %bb.0: ; %main_body 237; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec 238; GFX9-W64-NEXT: s_wqm_b64 exec, exec 239; GFX9-W64-NEXT: v_mov_b32_e32 v0, s0 240; GFX9-W64-NEXT: v_mov_b32_e32 v1, s1 241; GFX9-W64-NEXT: buffer_load_dword v0, v0, s[0:3], 0 idxen 242; GFX9-W64-NEXT: s_nop 0 243; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 244; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 245; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v1 246; GFX9-W64-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec killed $exec 247; GFX9-W64-NEXT: s_and_b64 exec, exec, s[2:3] 248; GFX9-W64-NEXT: ; return to shader part epilog 249; 250; GFX10-W32-LABEL: test6: 251; GFX10-W32: ; %bb.0: ; %main_body 252; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo 253; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 254; GFX10-W32-NEXT: v_mov_b32_e32 v0, s0 255; GFX10-W32-NEXT: v_mov_b32_e32 v1, s1 256; GFX10-W32-NEXT: s_clause 0x1 257; GFX10-W32-NEXT: buffer_load_dword v0, v0, s[0:3], 0 idxen 258; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 259; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 260; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v1 261; GFX10-W32-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec killed $exec 262; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s2 263; GFX10-W32-NEXT: ; return to shader part epilog 264main_body: 265 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 266 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 267 %out = fadd float %src0, %src1 268 %out.0 = bitcast float %out to i32 269 %out.1 = call i32 @llvm.amdgcn.wqm.i32(i32 %out.0) 270 %out.2 = bitcast i32 %out.1 to float 271 ret float %out.2 272} 273 274; NOTE: llvm.amdgcn.wwm is deprecated, use llvm.amdgcn.strict.wwm instead. 275 276; Check that WWM is triggered by the wwm intrinsic. 277define amdgpu_ps float @test_wwm1(i32 inreg %idx0, i32 inreg %idx1) { 278; GFX9-W64-LABEL: test_wwm1: 279; GFX9-W64: ; %bb.0: ; %main_body 280; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1 281; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 282; GFX9-W64-NEXT: v_mov_b32_e32 v2, s1 283; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 284; GFX9-W64-NEXT: s_nop 0 285; GFX9-W64-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen 286; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 287; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v2 288; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 289; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 290; GFX9-W64-NEXT: ; return to shader part epilog 291; 292; GFX10-W32-LABEL: test_wwm1: 293; GFX10-W32: ; %bb.0: ; %main_body 294; GFX10-W32-NEXT: s_or_saveexec_b32 s2, -1 295; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 296; GFX10-W32-NEXT: v_mov_b32_e32 v2, s1 297; GFX10-W32-NEXT: s_clause 0x1 298; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 299; GFX10-W32-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen 300; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 301; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v2 302; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2 303; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 304; GFX10-W32-NEXT: ; return to shader part epilog 305main_body: 306 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 307 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 308 %out = fadd float %src0, %src1 309 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out) 310 ret float %out.0 311} 312 313; Same as above, but with an integer type. 314define amdgpu_ps float @test_wwm2(i32 inreg %idx0, i32 inreg %idx1) { 315; GFX9-W64-LABEL: test_wwm2: 316; GFX9-W64: ; %bb.0: ; %main_body 317; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1 318; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 319; GFX9-W64-NEXT: v_mov_b32_e32 v2, s1 320; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 321; GFX9-W64-NEXT: s_nop 0 322; GFX9-W64-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen 323; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 324; GFX9-W64-NEXT: v_add_u32_e32 v1, v1, v2 325; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 326; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 327; GFX9-W64-NEXT: ; return to shader part epilog 328; 329; GFX10-W32-LABEL: test_wwm2: 330; GFX10-W32: ; %bb.0: ; %main_body 331; GFX10-W32-NEXT: s_or_saveexec_b32 s2, -1 332; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 333; GFX10-W32-NEXT: v_mov_b32_e32 v2, s1 334; GFX10-W32-NEXT: s_clause 0x1 335; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 336; GFX10-W32-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen 337; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 338; GFX10-W32-NEXT: v_add_nc_u32_e32 v1, v1, v2 339; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2 340; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 341; GFX10-W32-NEXT: ; return to shader part epilog 342main_body: 343 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 344 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 345 %src0.0 = bitcast float %src0 to i32 346 %src1.0 = bitcast float %src1 to i32 347 %out = add i32 %src0.0, %src1.0 348 %out.0 = call i32 @llvm.amdgcn.wwm.i32(i32 %out) 349 %out.1 = bitcast i32 %out.0 to float 350 ret float %out.1 351} 352 353; Check that we don't leave WWM on for computations that don't require WWM, 354; since that will lead clobbering things that aren't supposed to be clobbered 355; in cases like this. 356; We enforce this by checking that v_add gets emitted in the same block as 357; WWM computations. 358define amdgpu_ps float @test_wwm3(i32 inreg %idx) { 359; GFX9-W64-LABEL: test_wwm3: 360; GFX9-W64: ; %bb.0: ; %main_body 361; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 362; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 363; GFX9-W64-NEXT: v_cmp_gt_u32_e32 vcc, 16, v0 364; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 365; GFX9-W64-NEXT: s_and_saveexec_b64 s[2:3], vcc 366; GFX9-W64-NEXT: s_cbranch_execz .LBB9_2 367; GFX9-W64-NEXT: ; %bb.1: ; %if 368; GFX9-W64-NEXT: s_or_saveexec_b64 s[4:5], -1 369; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 370; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 371; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 372; GFX9-W64-NEXT: v_add_f32_e32 v2, v1, v1 373; GFX9-W64-NEXT: s_mov_b64 exec, s[4:5] 374; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2 375; GFX9-W64-NEXT: v_add_f32_e32 v0, v1, v0 376; GFX9-W64-NEXT: .LBB9_2: ; %endif 377; GFX9-W64-NEXT: s_or_b64 exec, exec, s[2:3] 378; GFX9-W64-NEXT: ; return to shader part epilog 379; 380; GFX10-W32-LABEL: test_wwm3: 381; GFX10-W32: ; %bb.0: ; %main_body 382; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 383; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 384; GFX10-W32-NEXT: v_cmp_gt_u32_e32 vcc_lo, 16, v0 385; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 386; GFX10-W32-NEXT: s_and_saveexec_b32 s1, vcc_lo 387; GFX10-W32-NEXT: s_cbranch_execz .LBB9_2 388; GFX10-W32-NEXT: ; %bb.1: ; %if 389; GFX10-W32-NEXT: s_or_saveexec_b32 s2, -1 390; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 391; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 392; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 393; GFX10-W32-NEXT: v_add_f32_e32 v2, v1, v1 394; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2 395; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2 396; GFX10-W32-NEXT: v_add_f32_e32 v0, v1, v0 397; GFX10-W32-NEXT: .LBB9_2: ; %endif 398; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s1 399; GFX10-W32-NEXT: ; return to shader part epilog 400main_body: 401 ; use mbcnt to make sure the branch is divergent 402 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 403 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 404 %cc = icmp uge i32 %hi, 16 405 br i1 %cc, label %endif, label %if 406 407if: 408 %src = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 409 %out = fadd float %src, %src 410 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out) 411 %out.1 = fadd float %src, %out.0 412 br label %endif 413 414endif: 415 %out.2 = phi float [ %out.1, %if ], [ 0.0, %main_body ] 416 ret float %out.2 417} 418 419; Check that WWM writes aren't coalesced with non-WWM writes, since the WWM 420; write could clobber disabled channels in the non-WWM one. 421; We enforce this by checking that v_mov gets emitted in the same block as 422; WWM computations. 423define amdgpu_ps float @test_wwm4(i32 inreg %idx) { 424; GFX9-W64-LABEL: test_wwm4: 425; GFX9-W64: ; %bb.0: ; %main_body 426; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 427; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 428; GFX9-W64-NEXT: v_cmp_gt_u32_e32 vcc, 16, v0 429; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 430; GFX9-W64-NEXT: s_and_saveexec_b64 s[2:3], vcc 431; GFX9-W64-NEXT: s_cbranch_execz .LBB10_2 432; GFX9-W64-NEXT: ; %bb.1: ; %if 433; GFX9-W64-NEXT: s_or_saveexec_b64 s[4:5], -1 434; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 435; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 436; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 437; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v1 438; GFX9-W64-NEXT: s_mov_b64 exec, s[4:5] 439; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 440; GFX9-W64-NEXT: .LBB10_2: ; %endif 441; GFX9-W64-NEXT: s_or_b64 exec, exec, s[2:3] 442; GFX9-W64-NEXT: ; return to shader part epilog 443; 444; GFX10-W32-LABEL: test_wwm4: 445; GFX10-W32: ; %bb.0: ; %main_body 446; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 447; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 448; GFX10-W32-NEXT: v_cmp_gt_u32_e32 vcc_lo, 16, v0 449; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 450; GFX10-W32-NEXT: s_and_saveexec_b32 s1, vcc_lo 451; GFX10-W32-NEXT: s_cbranch_execz .LBB10_2 452; GFX10-W32-NEXT: ; %bb.1: ; %if 453; GFX10-W32-NEXT: s_or_saveexec_b32 s2, -1 454; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 455; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 456; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 457; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v1 458; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2 459; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 460; GFX10-W32-NEXT: .LBB10_2: ; %endif 461; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s1 462; GFX10-W32-NEXT: ; return to shader part epilog 463main_body: 464 ; use mbcnt to make sure the branch is divergent 465 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 466 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 467 %cc = icmp uge i32 %hi, 16 468 br i1 %cc, label %endif, label %if 469 470if: 471 %src = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 472 %out = fadd float %src, %src 473 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out) 474 br label %endif 475 476endif: 477 %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ] 478 ret float %out.1 479} 480 481; Make sure the transition from Exact to WWM then WQM works properly. 482define amdgpu_ps float @test_wwm5(i32 inreg %idx0, i32 inreg %idx1) { 483; GFX9-W64-LABEL: test_wwm5: 484; GFX9-W64: ; %bb.0: ; %main_body 485; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec 486; GFX9-W64-NEXT: v_mov_b32_e32 v0, s0 487; GFX9-W64-NEXT: buffer_load_dword v2, v0, s[0:3], 0 idxen 488; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 489; GFX9-W64-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen 490; GFX9-W64-NEXT: s_or_saveexec_b64 s[4:5], -1 491; GFX9-W64-NEXT: v_mov_b32_e32 v1, s1 492; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 493; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 494; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v1 495; GFX9-W64-NEXT: s_mov_b64 exec, s[4:5] 496; GFX9-W64-NEXT: s_wqm_b64 exec, exec 497; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 498; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v0 499; GFX9-W64-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec killed $exec 500; GFX9-W64-NEXT: s_and_b64 exec, exec, s[2:3] 501; GFX9-W64-NEXT: ; return to shader part epilog 502; 503; GFX10-W32-LABEL: test_wwm5: 504; GFX10-W32: ; %bb.0: ; %main_body 505; GFX10-W32-NEXT: v_mov_b32_e32 v0, s0 506; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo 507; GFX10-W32-NEXT: buffer_load_dword v2, v0, s[0:3], 0 idxen 508; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 509; GFX10-W32-NEXT: v_mov_b32_e32 v1, s1 510; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 511; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 512; GFX10-W32-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen 513; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 514; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 515; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 516; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v1 517; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 518; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 519; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 520; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v0 521; GFX10-W32-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec killed $exec 522; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s2 523; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 524; GFX10-W32-NEXT: ; return to shader part epilog 525main_body: 526 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 527 call void @llvm.amdgcn.struct.buffer.store.f32(float %src0, <4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 528 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 529 %temp = fadd float %src1, %src1 530 %temp.0 = call float @llvm.amdgcn.wwm.f32(float %temp) 531 %out = fadd float %temp.0, %temp.0 532 %out.0 = call float @llvm.amdgcn.wqm.f32(float %out) 533 ret float %out.0 534} 535 536; Check that WWM is turned on correctly across basic block boundaries. 537; if..then..endif version 538;SI-CHECK: buffer_load_dword 539;VI-CHECK: flat_load_dword 540;SI-CHECK: buffer_load_dword 541;VI-CHECK: flat_load_dword 542define amdgpu_ps float @test_wwm6_then() { 543; GFX9-W64-LABEL: test_wwm6_then: 544; GFX9-W64: ; %bb.0: ; %main_body 545; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], -1 546; GFX9-W64-NEXT: global_load_dword v1, v[3:4], off glc 547; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 548; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1] 549; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 550; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 551; GFX9-W64-NEXT: v_cmp_gt_u32_e32 vcc, 16, v0 552; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 553; GFX9-W64-NEXT: s_and_saveexec_b64 s[0:1], vcc 554; GFX9-W64-NEXT: s_cbranch_execz .LBB12_2 555; GFX9-W64-NEXT: ; %bb.1: ; %if 556; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1 557; GFX9-W64-NEXT: global_load_dword v2, v[3:4], off glc 558; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 559; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v2 560; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 561; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 562; GFX9-W64-NEXT: .LBB12_2: ; %endif 563; GFX9-W64-NEXT: s_or_b64 exec, exec, s[0:1] 564; GFX9-W64-NEXT: ; return to shader part epilog 565; 566; GFX10-W32-LABEL: test_wwm6_then: 567; GFX10-W32: ; %bb.0: ; %main_body 568; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 569; GFX10-W32-NEXT: global_load_dword v1, v[3:4], off glc dlc 570; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 571; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 572; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 573; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 574; GFX10-W32-NEXT: v_cmp_gt_u32_e32 vcc_lo, 16, v0 575; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 576; GFX10-W32-NEXT: s_and_saveexec_b32 s0, vcc_lo 577; GFX10-W32-NEXT: s_cbranch_execz .LBB12_2 578; GFX10-W32-NEXT: ; %bb.1: ; %if 579; GFX10-W32-NEXT: s_or_saveexec_b32 s1, -1 580; GFX10-W32-NEXT: global_load_dword v2, v[3:4], off glc dlc 581; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 582; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v2 583; GFX10-W32-NEXT: s_mov_b32 exec_lo, s1 584; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 585; GFX10-W32-NEXT: .LBB12_2: ; %endif 586; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s0 587; GFX10-W32-NEXT: ; return to shader part epilog 588main_body: 589 %src0 = load volatile float, float addrspace(1)* undef 590 ; use mbcnt to make sure the branch is divergent 591 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 592 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 593 %cc = icmp uge i32 %hi, 16 594 br i1 %cc, label %endif, label %if 595 596if: 597 %src1 = load volatile float, float addrspace(1)* undef 598 %out = fadd float %src0, %src1 599 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out) 600 br label %endif 601 602endif: 603 %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ] 604 ret float %out.1 605} 606 607; Check that WWM is turned on correctly across basic block boundaries. 608; loop version 609;SI-CHECK: buffer_load_dword 610;VI-CHECK: flat_load_dword 611;SI-CHECK: buffer_load_dword 612;VI-CHECK: flat_load_dword 613define amdgpu_ps float @test_wwm6_loop() { 614; GFX9-W64-LABEL: test_wwm6_loop: 615; GFX9-W64: ; %bb.0: ; %main_body 616; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], -1 617; GFX9-W64-NEXT: global_load_dword v1, v[3:4], off glc 618; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 619; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1] 620; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 621; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v0 622; GFX9-W64-NEXT: s_mov_b64 s[0:1], 0 623; GFX9-W64-NEXT: .LBB13_1: ; %loop 624; GFX9-W64-NEXT: ; =>This Inner Loop Header: Depth=1 625; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1 626; GFX9-W64-NEXT: global_load_dword v2, v[3:4], off glc 627; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 628; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 629; GFX9-W64-NEXT: v_add_u32_e32 v3, -1, v3 630; GFX9-W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3 631; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1 632; GFX9-W64-NEXT: v_add_f32_e32 v2, v1, v2 633; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 634; GFX9-W64-NEXT: s_or_b64 s[0:1], vcc, s[0:1] 635; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2 636; GFX9-W64-NEXT: s_andn2_b64 exec, exec, s[0:1] 637; GFX9-W64-NEXT: s_cbranch_execnz .LBB13_1 638; GFX9-W64-NEXT: ; %bb.2: ; %endloop 639; GFX9-W64-NEXT: s_or_b64 exec, exec, s[0:1] 640; GFX9-W64-NEXT: ; return to shader part epilog 641; 642; GFX10-W32-LABEL: test_wwm6_loop: 643; GFX10-W32: ; %bb.0: ; %main_body 644; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 645; GFX10-W32-NEXT: global_load_dword v1, v[3:4], off glc dlc 646; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 647; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 648; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 649; GFX10-W32-NEXT: s_mov_b32 s0, 0 650; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v0 651; GFX10-W32-NEXT: .LBB13_1: ; %loop 652; GFX10-W32-NEXT: ; =>This Inner Loop Header: Depth=1 653; GFX10-W32-NEXT: s_or_saveexec_b32 s1, -1 654; GFX10-W32-NEXT: global_load_dword v2, v[3:4], off glc dlc 655; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 656; GFX10-W32-NEXT: s_mov_b32 exec_lo, s1 657; GFX10-W32-NEXT: v_add_nc_u32_e32 v3, -1, v3 658; GFX10-W32-NEXT: s_or_saveexec_b32 s1, -1 659; GFX10-W32-NEXT: v_add_f32_e32 v2, v1, v2 660; GFX10-W32-NEXT: s_mov_b32 exec_lo, s1 661; GFX10-W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v3 662; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2 663; GFX10-W32-NEXT: s_or_b32 s0, vcc_lo, s0 664; GFX10-W32-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 665; GFX10-W32-NEXT: s_cbranch_execnz .LBB13_1 666; GFX10-W32-NEXT: ; %bb.2: ; %endloop 667; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s0 668; GFX10-W32-NEXT: ; return to shader part epilog 669main_body: 670 %src0 = load volatile float, float addrspace(1)* undef 671 ; use mbcnt to make sure the branch is divergent 672 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 673 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 674 br label %loop 675 676loop: 677 %counter = phi i32 [ %hi, %main_body ], [ %counter.1, %loop ] 678 %src1 = load volatile float, float addrspace(1)* undef 679 %out = fadd float %src0, %src1 680 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out) 681 %counter.1 = sub i32 %counter, 1 682 %cc = icmp ne i32 %counter.1, 0 683 br i1 %cc, label %loop, label %endloop 684 685endloop: 686 ret float %out.0 687} 688 689; Check that @llvm.amdgcn.set.inactive disables WWM. 690define amdgpu_ps void @test_wwm_set_inactive1(i32 inreg %idx) { 691; GFX9-W64-LABEL: test_wwm_set_inactive1: 692; GFX9-W64: ; %bb.0: ; %main_body 693; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 694; GFX9-W64-NEXT: buffer_load_dword v0, v1, s[0:3], 0 idxen 695; GFX9-W64-NEXT: s_not_b64 exec, exec 696; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 697; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 698; GFX9-W64-NEXT: s_not_b64 exec, exec 699; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], -1 700; GFX9-W64-NEXT: v_add_u32_e32 v0, v0, v0 701; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1] 702; GFX9-W64-NEXT: v_mov_b32_e32 v2, v0 703; GFX9-W64-NEXT: buffer_store_dword v2, v1, s[0:3], 0 idxen 704; GFX9-W64-NEXT: s_endpgm 705; 706; GFX10-W32-LABEL: test_wwm_set_inactive1: 707; GFX10-W32: ; %bb.0: ; %main_body 708; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 709; GFX10-W32-NEXT: buffer_load_dword v0, v1, s[0:3], 0 idxen 710; GFX10-W32-NEXT: s_not_b32 exec_lo, exec_lo 711; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 712; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 713; GFX10-W32-NEXT: s_not_b32 exec_lo, exec_lo 714; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 715; GFX10-W32-NEXT: v_add_nc_u32_e32 v0, v0, v0 716; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 717; GFX10-W32-NEXT: v_mov_b32_e32 v2, v0 718; GFX10-W32-NEXT: buffer_store_dword v2, v1, s[0:3], 0 idxen 719; GFX10-W32-NEXT: s_endpgm 720main_body: 721 %src = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 722 %src.0 = bitcast float %src to i32 723 %src.1 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %src.0, i32 0) 724 %out = add i32 %src.1, %src.1 725 %out.0 = call i32 @llvm.amdgcn.wwm.i32(i32 %out) 726 %out.1 = bitcast i32 %out.0 to float 727 call void @llvm.amdgcn.struct.buffer.store.f32(float %out.1, <4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 728 ret void 729} 730 731; Check that Strict WQM is triggered by the strict_wqm intrinsic. 732define amdgpu_ps float @test_strict_wqm1(i32 inreg %idx0, i32 inreg %idx1) { 733; GFX9-W64-LABEL: test_strict_wqm1: 734; GFX9-W64: ; %bb.0: ; %main_body 735; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec 736; GFX9-W64-NEXT: s_wqm_b64 exec, exec 737; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 738; GFX9-W64-NEXT: v_mov_b32_e32 v2, s1 739; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 740; GFX9-W64-NEXT: s_nop 0 741; GFX9-W64-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen 742; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 743; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v2 744; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 745; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 746; GFX9-W64-NEXT: ; return to shader part epilog 747; 748; GFX10-W32-LABEL: test_strict_wqm1: 749; GFX10-W32: ; %bb.0: ; %main_body 750; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo 751; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 752; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 753; GFX10-W32-NEXT: v_mov_b32_e32 v2, s1 754; GFX10-W32-NEXT: s_clause 0x1 755; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 756; GFX10-W32-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen 757; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 758; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v2 759; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2 760; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 761; GFX10-W32-NEXT: ; return to shader part epilog 762main_body: 763 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 764 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 765 %out = fadd float %src0, %src1 766 %out.0 = call float @llvm.amdgcn.strict.wqm.f32(float %out) 767 ret float %out.0 768} 769 770; Same as above, but with an integer type. 771define amdgpu_ps float @test_strict_wqm2(i32 inreg %idx0, i32 inreg %idx1) { 772; GFX9-W64-LABEL: test_strict_wqm2: 773; GFX9-W64: ; %bb.0: ; %main_body 774; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec 775; GFX9-W64-NEXT: s_wqm_b64 exec, exec 776; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 777; GFX9-W64-NEXT: v_mov_b32_e32 v2, s1 778; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 779; GFX9-W64-NEXT: s_nop 0 780; GFX9-W64-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen 781; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 782; GFX9-W64-NEXT: v_add_u32_e32 v1, v1, v2 783; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 784; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 785; GFX9-W64-NEXT: ; return to shader part epilog 786; 787; GFX10-W32-LABEL: test_strict_wqm2: 788; GFX10-W32: ; %bb.0: ; %main_body 789; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo 790; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 791; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 792; GFX10-W32-NEXT: v_mov_b32_e32 v2, s1 793; GFX10-W32-NEXT: s_clause 0x1 794; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 795; GFX10-W32-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen 796; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 797; GFX10-W32-NEXT: v_add_nc_u32_e32 v1, v1, v2 798; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2 799; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 800; GFX10-W32-NEXT: ; return to shader part epilog 801main_body: 802 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 803 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 804 %src0.0 = bitcast float %src0 to i32 805 %src1.0 = bitcast float %src1 to i32 806 %out = add i32 %src0.0, %src1.0 807 %out.0 = call i32 @llvm.amdgcn.strict.wqm.i32(i32 %out) 808 %out.1 = bitcast i32 %out.0 to float 809 ret float %out.1 810} 811 812; Check that we don't leave Strict WQM on for computations that don't require it, 813; since that will lead clobbering things that aren't supposed to be clobbered 814; in cases like this. 815; We enforce this by checking that v_add gets emitted in the same block as 816; WWM computations. 817define amdgpu_ps float @test_strict_wqm3(i32 inreg %idx) { 818; GFX9-W64-LABEL: test_strict_wqm3: 819; GFX9-W64: ; %bb.0: ; %main_body 820; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 821; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 822; GFX9-W64-NEXT: v_cmp_gt_u32_e32 vcc, 16, v0 823; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 824; GFX9-W64-NEXT: s_and_saveexec_b64 s[2:3], vcc 825; GFX9-W64-NEXT: s_cbranch_execz .LBB17_2 826; GFX9-W64-NEXT: ; %bb.1: ; %if 827; GFX9-W64-NEXT: s_mov_b64 s[4:5], exec 828; GFX9-W64-NEXT: s_wqm_b64 exec, exec 829; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 830; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 831; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 832; GFX9-W64-NEXT: v_add_f32_e32 v2, v1, v1 833; GFX9-W64-NEXT: s_mov_b64 exec, s[4:5] 834; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2 835; GFX9-W64-NEXT: v_add_f32_e32 v0, v1, v0 836; GFX9-W64-NEXT: .LBB17_2: ; %endif 837; GFX9-W64-NEXT: s_or_b64 exec, exec, s[2:3] 838; GFX9-W64-NEXT: ; return to shader part epilog 839; 840; GFX10-W32-LABEL: test_strict_wqm3: 841; GFX10-W32: ; %bb.0: ; %main_body 842; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 843; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 844; GFX10-W32-NEXT: v_cmp_gt_u32_e32 vcc_lo, 16, v0 845; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 846; GFX10-W32-NEXT: s_and_saveexec_b32 s1, vcc_lo 847; GFX10-W32-NEXT: s_cbranch_execz .LBB17_2 848; GFX10-W32-NEXT: ; %bb.1: ; %if 849; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo 850; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 851; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 852; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 853; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 854; GFX10-W32-NEXT: v_add_f32_e32 v2, v1, v1 855; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2 856; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2 857; GFX10-W32-NEXT: v_add_f32_e32 v0, v1, v0 858; GFX10-W32-NEXT: .LBB17_2: ; %endif 859; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s1 860; GFX10-W32-NEXT: ; return to shader part epilog 861main_body: 862 ; use mbcnt to make sure the branch is divergent 863 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 864 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 865 %cc = icmp uge i32 %hi, 16 866 br i1 %cc, label %endif, label %if 867 868if: 869 %src = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 870 %out = fadd float %src, %src 871 %out.0 = call float @llvm.amdgcn.strict.wqm.f32(float %out) 872 %out.1 = fadd float %src, %out.0 873 br label %endif 874 875endif: 876 %out.2 = phi float [ %out.1, %if ], [ 0.0, %main_body ] 877 ret float %out.2 878} 879 880; Check that Strict WQM writes aren't coalesced with non-strict writes, since 881; the Strict WQM write could clobber disabled channels in the non-strict one. 882; We enforce this by checking that v_mov gets emitted in the same block as 883; WWM computations. 884define amdgpu_ps float @test_strict_wqm4(i32 inreg %idx) { 885; GFX9-W64-LABEL: test_strict_wqm4: 886; GFX9-W64: ; %bb.0: ; %main_body 887; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 888; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 889; GFX9-W64-NEXT: v_cmp_gt_u32_e32 vcc, 16, v0 890; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 891; GFX9-W64-NEXT: s_and_saveexec_b64 s[2:3], vcc 892; GFX9-W64-NEXT: s_cbranch_execz .LBB18_2 893; GFX9-W64-NEXT: ; %bb.1: ; %if 894; GFX9-W64-NEXT: s_mov_b64 s[4:5], exec 895; GFX9-W64-NEXT: s_wqm_b64 exec, exec 896; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 897; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 898; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 899; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v1 900; GFX9-W64-NEXT: s_mov_b64 exec, s[4:5] 901; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 902; GFX9-W64-NEXT: .LBB18_2: ; %endif 903; GFX9-W64-NEXT: s_or_b64 exec, exec, s[2:3] 904; GFX9-W64-NEXT: ; return to shader part epilog 905; 906; GFX10-W32-LABEL: test_strict_wqm4: 907; GFX10-W32: ; %bb.0: ; %main_body 908; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 909; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 910; GFX10-W32-NEXT: v_cmp_gt_u32_e32 vcc_lo, 16, v0 911; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 912; GFX10-W32-NEXT: s_and_saveexec_b32 s1, vcc_lo 913; GFX10-W32-NEXT: s_cbranch_execz .LBB18_2 914; GFX10-W32-NEXT: ; %bb.1: ; %if 915; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo 916; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 917; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 918; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 919; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 920; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v1 921; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2 922; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 923; GFX10-W32-NEXT: .LBB18_2: ; %endif 924; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s1 925; GFX10-W32-NEXT: ; return to shader part epilog 926main_body: 927 ; use mbcnt to make sure the branch is divergent 928 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 929 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 930 %cc = icmp uge i32 %hi, 16 931 br i1 %cc, label %endif, label %if 932 933if: 934 %src = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 935 %out = fadd float %src, %src 936 %out.0 = call float @llvm.amdgcn.strict.wqm.f32(float %out) 937 br label %endif 938 939endif: 940 %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ] 941 ret float %out.1 942} 943 944; Make sure the transition from Exact to Strict WQM then WQM works properly. 945define amdgpu_ps float @test_strict_wqm5(i32 inreg %idx0, i32 inreg %idx1) { 946; GFX9-W64-LABEL: test_strict_wqm5: 947; GFX9-W64: ; %bb.0: ; %main_body 948; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec 949; GFX9-W64-NEXT: v_mov_b32_e32 v0, s0 950; GFX9-W64-NEXT: buffer_load_dword v2, v0, s[0:3], 0 idxen 951; GFX9-W64-NEXT: s_mov_b64 s[4:5], exec 952; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 953; GFX9-W64-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen 954; GFX9-W64-NEXT: s_wqm_b64 exec, exec 955; GFX9-W64-NEXT: v_mov_b32_e32 v1, s1 956; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 957; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 958; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v1 959; GFX9-W64-NEXT: s_mov_b64 exec, s[4:5] 960; GFX9-W64-NEXT: s_wqm_b64 exec, exec 961; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 962; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v0 963; GFX9-W64-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec killed $exec 964; GFX9-W64-NEXT: s_and_b64 exec, exec, s[2:3] 965; GFX9-W64-NEXT: ; return to shader part epilog 966; 967; GFX10-W32-LABEL: test_strict_wqm5: 968; GFX10-W32: ; %bb.0: ; %main_body 969; GFX10-W32-NEXT: v_mov_b32_e32 v0, s0 970; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo 971; GFX10-W32-NEXT: buffer_load_dword v2, v0, s[0:3], 0 idxen 972; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo 973; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 974; GFX10-W32-NEXT: v_mov_b32_e32 v1, s1 975; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 976; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 977; GFX10-W32-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen 978; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo 979; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 980; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 981; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 982; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v1 983; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 984; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 985; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 986; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v0 987; GFX10-W32-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec killed $exec 988; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s2 989; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 990; GFX10-W32-NEXT: ; return to shader part epilog 991main_body: 992 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 993 call void @llvm.amdgcn.struct.buffer.store.f32(float %src0, <4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 994 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 995 %temp = fadd float %src1, %src1 996 %temp.0 = call float @llvm.amdgcn.strict.wqm.f32(float %temp) 997 %out = fadd float %temp.0, %temp.0 998 %out.0 = call float @llvm.amdgcn.wqm.f32(float %out) 999 ret float %out.0 1000} 1001 1002; Check that Strict WQM is turned on correctly across basic block boundaries. 1003; if..then..endif version 1004;SI-CHECK: buffer_load_dword 1005;VI-CHECK: flat_load_dword 1006;SI-CHECK: buffer_load_dword 1007;VI-CHECK: flat_load_dword 1008define amdgpu_ps float @test_strict_wqm6_then() { 1009; GFX9-W64-LABEL: test_strict_wqm6_then: 1010; GFX9-W64: ; %bb.0: ; %main_body 1011; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec 1012; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1013; GFX9-W64-NEXT: global_load_dword v1, v[3:4], off glc 1014; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1015; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1] 1016; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 1017; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 1018; GFX9-W64-NEXT: v_cmp_gt_u32_e32 vcc, 16, v0 1019; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 1020; GFX9-W64-NEXT: s_and_saveexec_b64 s[0:1], vcc 1021; GFX9-W64-NEXT: s_cbranch_execz .LBB20_2 1022; GFX9-W64-NEXT: ; %bb.1: ; %if 1023; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec 1024; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1025; GFX9-W64-NEXT: global_load_dword v2, v[3:4], off glc 1026; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1027; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v2 1028; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 1029; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 1030; GFX9-W64-NEXT: .LBB20_2: ; %endif 1031; GFX9-W64-NEXT: s_or_b64 exec, exec, s[0:1] 1032; GFX9-W64-NEXT: ; return to shader part epilog 1033; 1034; GFX10-W32-LABEL: test_strict_wqm6_then: 1035; GFX10-W32: ; %bb.0: ; %main_body 1036; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo 1037; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1038; GFX10-W32-NEXT: global_load_dword v1, v[3:4], off glc dlc 1039; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1040; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 1041; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 1042; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 1043; GFX10-W32-NEXT: v_cmp_gt_u32_e32 vcc_lo, 16, v0 1044; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 1045; GFX10-W32-NEXT: s_and_saveexec_b32 s0, vcc_lo 1046; GFX10-W32-NEXT: s_cbranch_execz .LBB20_2 1047; GFX10-W32-NEXT: ; %bb.1: ; %if 1048; GFX10-W32-NEXT: s_mov_b32 s1, exec_lo 1049; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1050; GFX10-W32-NEXT: global_load_dword v2, v[3:4], off glc dlc 1051; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1052; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v2 1053; GFX10-W32-NEXT: s_mov_b32 exec_lo, s1 1054; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 1055; GFX10-W32-NEXT: .LBB20_2: ; %endif 1056; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s0 1057; GFX10-W32-NEXT: ; return to shader part epilog 1058main_body: 1059 %src0 = load volatile float, float addrspace(1)* undef 1060 ; use mbcnt to make sure the branch is divergent 1061 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 1062 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 1063 %cc = icmp uge i32 %hi, 16 1064 br i1 %cc, label %endif, label %if 1065 1066if: 1067 %src1 = load volatile float, float addrspace(1)* undef 1068 %out = fadd float %src0, %src1 1069 %out.0 = call float @llvm.amdgcn.strict.wqm.f32(float %out) 1070 br label %endif 1071 1072endif: 1073 %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ] 1074 ret float %out.1 1075} 1076 1077; Check that Strict WQM is turned on correctly across basic block boundaries. 1078; loop version 1079;SI-CHECK: buffer_load_dword 1080;VI-CHECK: flat_load_dword 1081;SI-CHECK: buffer_load_dword 1082;VI-CHECK: flat_load_dword 1083define amdgpu_ps float @test_strict_wqm6_loop() { 1084; GFX9-W64-LABEL: test_strict_wqm6_loop: 1085; GFX9-W64: ; %bb.0: ; %main_body 1086; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec 1087; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1088; GFX9-W64-NEXT: global_load_dword v1, v[3:4], off glc 1089; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1090; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1] 1091; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 1092; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v0 1093; GFX9-W64-NEXT: s_mov_b64 s[0:1], 0 1094; GFX9-W64-NEXT: .LBB21_1: ; %loop 1095; GFX9-W64-NEXT: ; =>This Inner Loop Header: Depth=1 1096; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec 1097; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1098; GFX9-W64-NEXT: global_load_dword v2, v[3:4], off glc 1099; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1100; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 1101; GFX9-W64-NEXT: v_add_u32_e32 v3, -1, v3 1102; GFX9-W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3 1103; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec 1104; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1105; GFX9-W64-NEXT: v_add_f32_e32 v2, v1, v2 1106; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 1107; GFX9-W64-NEXT: s_or_b64 s[0:1], vcc, s[0:1] 1108; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2 1109; GFX9-W64-NEXT: s_andn2_b64 exec, exec, s[0:1] 1110; GFX9-W64-NEXT: s_cbranch_execnz .LBB21_1 1111; GFX9-W64-NEXT: ; %bb.2: ; %endloop 1112; GFX9-W64-NEXT: s_or_b64 exec, exec, s[0:1] 1113; GFX9-W64-NEXT: ; return to shader part epilog 1114; 1115; GFX10-W32-LABEL: test_strict_wqm6_loop: 1116; GFX10-W32: ; %bb.0: ; %main_body 1117; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo 1118; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1119; GFX10-W32-NEXT: global_load_dword v1, v[3:4], off glc dlc 1120; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1121; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 1122; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 1123; GFX10-W32-NEXT: s_mov_b32 s0, 0 1124; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v0 1125; GFX10-W32-NEXT: .LBB21_1: ; %loop 1126; GFX10-W32-NEXT: ; =>This Inner Loop Header: Depth=1 1127; GFX10-W32-NEXT: s_mov_b32 s1, exec_lo 1128; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1129; GFX10-W32-NEXT: global_load_dword v2, v[3:4], off glc dlc 1130; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1131; GFX10-W32-NEXT: s_mov_b32 exec_lo, s1 1132; GFX10-W32-NEXT: v_add_nc_u32_e32 v3, -1, v3 1133; GFX10-W32-NEXT: s_mov_b32 s1, exec_lo 1134; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1135; GFX10-W32-NEXT: v_add_f32_e32 v2, v1, v2 1136; GFX10-W32-NEXT: s_mov_b32 exec_lo, s1 1137; GFX10-W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v3 1138; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2 1139; GFX10-W32-NEXT: s_or_b32 s0, vcc_lo, s0 1140; GFX10-W32-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 1141; GFX10-W32-NEXT: s_cbranch_execnz .LBB21_1 1142; GFX10-W32-NEXT: ; %bb.2: ; %endloop 1143; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s0 1144; GFX10-W32-NEXT: ; return to shader part epilog 1145main_body: 1146 %src0 = load volatile float, float addrspace(1)* undef 1147 ; use mbcnt to make sure the branch is divergent 1148 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 1149 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 1150 br label %loop 1151 1152loop: 1153 %counter = phi i32 [ %hi, %main_body ], [ %counter.1, %loop ] 1154 %src1 = load volatile float, float addrspace(1)* undef 1155 %out = fadd float %src0, %src1 1156 %out.0 = call float @llvm.amdgcn.strict.wqm.f32(float %out) 1157 %counter.1 = sub i32 %counter, 1 1158 %cc = icmp ne i32 %counter.1, 0 1159 br i1 %cc, label %loop, label %endloop 1160 1161endloop: 1162 ret float %out.0 1163} 1164 1165; Check that enabling WQM anywhere enables WQM for the set.inactive source. 1166define amdgpu_ps void @test_set_inactive2(i32 inreg %idx0, i32 inreg %idx1) { 1167; GFX9-W64-LABEL: test_set_inactive2: 1168; GFX9-W64: ; %bb.0: ; %main_body 1169; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec 1170; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1171; GFX9-W64-NEXT: v_mov_b32_e32 v0, s1 1172; GFX9-W64-NEXT: v_mov_b32_e32 v2, s0 1173; GFX9-W64-NEXT: buffer_load_dword v1, v0, s[0:3], 0 idxen 1174; GFX9-W64-NEXT: s_nop 0 1175; GFX9-W64-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen 1176; GFX9-W64-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $scc killed $exec 1177; GFX9-W64-NEXT: ; kill: def $vgpr2 killed $vgpr2 killed $exec killed $exec 1178; GFX9-W64-NEXT: s_and_b64 exec, exec, s[2:3] 1179; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1180; GFX9-W64-NEXT: v_add_u32_e32 v1, v2, v1 1181; GFX9-W64-NEXT: buffer_store_dword v1, v0, s[0:3], 0 idxen 1182; GFX9-W64-NEXT: s_endpgm 1183; 1184; GFX10-W32-LABEL: test_set_inactive2: 1185; GFX10-W32: ; %bb.0: ; %main_body 1186; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo 1187; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1188; GFX10-W32-NEXT: v_mov_b32_e32 v0, s1 1189; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 1190; GFX10-W32-NEXT: s_clause 0x1 1191; GFX10-W32-NEXT: buffer_load_dword v2, v0, s[0:3], 0 idxen 1192; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 1193; GFX10-W32-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $scc killed $exec 1194; GFX10-W32-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $exec killed $exec 1195; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s2 1196; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1197; GFX10-W32-NEXT: v_add_nc_u32_e32 v1, v1, v2 1198; GFX10-W32-NEXT: buffer_store_dword v1, v0, s[0:3], 0 idxen 1199; GFX10-W32-NEXT: s_endpgm 1200main_body: 1201 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 1202 %src1.0 = bitcast float %src1 to i32 1203 %src1.1 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %src1.0, i32 undef) 1204 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 1205 %src0.0 = bitcast float %src0 to i32 1206 %src0.1 = call i32 @llvm.amdgcn.wqm.i32(i32 %src0.0) 1207 %out = add i32 %src0.1, %src1.1 1208 %out.0 = bitcast i32 %out to float 1209 call void @llvm.amdgcn.struct.buffer.store.f32(float %out.0, <4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 1210 ret void 1211} 1212 1213; Check a case of one branch of an if-else requiring WQM, the other requiring 1214; exact. 1215; Note: In this particular case, the save-and-restore could be avoided if the 1216; analysis understood that the two branches of the if-else are mutually 1217; exclusive. 1218define amdgpu_ps float @test_control_flow_0(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data) { 1219; GFX9-W64-LABEL: test_control_flow_0: 1220; GFX9-W64: ; %bb.0: ; %main_body 1221; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec 1222; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1223; GFX9-W64-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1 1224; GFX9-W64-NEXT: s_and_saveexec_b64 s[14:15], vcc 1225; GFX9-W64-NEXT: s_xor_b64 s[14:15], exec, s[14:15] 1226; GFX9-W64-NEXT: s_cbranch_execz .LBB23_2 1227; GFX9-W64-NEXT: ; %bb.1: ; %ELSE 1228; GFX9-W64-NEXT: s_and_saveexec_b64 s[16:17], s[12:13] 1229; GFX9-W64-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen 1230; GFX9-W64-NEXT: ; implicit-def: $vgpr0 1231; GFX9-W64-NEXT: s_mov_b64 exec, s[16:17] 1232; GFX9-W64-NEXT: .LBB23_2: ; %Flow 1233; GFX9-W64-NEXT: s_or_saveexec_b64 s[14:15], s[14:15] 1234; GFX9-W64-NEXT: s_xor_b64 exec, exec, s[14:15] 1235; GFX9-W64-NEXT: s_cbranch_execz .LBB23_4 1236; GFX9-W64-NEXT: ; %bb.3: ; %IF 1237; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 1238; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1239; GFX9-W64-NEXT: image_sample v2, v0, s[0:7], s[8:11] dmask:0x1 1240; GFX9-W64-NEXT: .LBB23_4: ; %END 1241; GFX9-W64-NEXT: s_or_b64 exec, exec, s[14:15] 1242; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 1243; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1244; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2 1245; GFX9-W64-NEXT: ; return to shader part epilog 1246; 1247; GFX10-W32-LABEL: test_control_flow_0: 1248; GFX10-W32: ; %bb.0: ; %main_body 1249; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo 1250; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1251; GFX10-W32-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1 1252; GFX10-W32-NEXT: s_and_saveexec_b32 s13, vcc_lo 1253; GFX10-W32-NEXT: s_xor_b32 s13, exec_lo, s13 1254; GFX10-W32-NEXT: s_cbranch_execz .LBB23_2 1255; GFX10-W32-NEXT: ; %bb.1: ; %ELSE 1256; GFX10-W32-NEXT: s_and_saveexec_b32 s14, s12 1257; GFX10-W32-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen 1258; GFX10-W32-NEXT: ; implicit-def: $vgpr0 1259; GFX10-W32-NEXT: s_mov_b32 exec_lo, s14 1260; GFX10-W32-NEXT: .LBB23_2: ; %Flow 1261; GFX10-W32-NEXT: s_or_saveexec_b32 s13, s13 1262; GFX10-W32-NEXT: s_xor_b32 exec_lo, exec_lo, s13 1263; GFX10-W32-NEXT: s_cbranch_execz .LBB23_4 1264; GFX10-W32-NEXT: ; %bb.3: ; %IF 1265; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 1266; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1267; GFX10-W32-NEXT: image_sample v2, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 1268; GFX10-W32-NEXT: .LBB23_4: ; %END 1269; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s13 1270; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 1271; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1272; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2 1273; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 1274; GFX10-W32-NEXT: ; return to shader part epilog 1275main_body: 1276 %cmp = icmp eq i32 %z, 0 1277 br i1 %cmp, label %IF, label %ELSE 1278 1279IF: 1280 %c.bc = bitcast i32 %c to float 1281 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1282 %tex0 = extractelement <4 x float> %tex, i32 0 1283 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1284 %data.if = extractelement <4 x float> %dtex, i32 0 1285 br label %END 1286 1287ELSE: 1288 call void @llvm.amdgcn.struct.buffer.store.f32(float %data, <4 x i32> undef, i32 %c, i32 0, i32 0, i32 0) 1289 br label %END 1290 1291END: 1292 %r = phi float [ %data.if, %IF ], [ %data, %ELSE ] 1293 ret float %r 1294} 1295 1296; Reverse branch order compared to the previous test. 1297define amdgpu_ps float @test_control_flow_1(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data) { 1298; GFX9-W64-LABEL: test_control_flow_1: 1299; GFX9-W64: ; %bb.0: ; %main_body 1300; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec 1301; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1302; GFX9-W64-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1 1303; GFX9-W64-NEXT: s_and_saveexec_b64 s[14:15], vcc 1304; GFX9-W64-NEXT: s_xor_b64 s[14:15], exec, s[14:15] 1305; GFX9-W64-NEXT: s_cbranch_execz .LBB24_2 1306; GFX9-W64-NEXT: ; %bb.1: ; %IF 1307; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 1308; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1309; GFX9-W64-NEXT: image_sample v2, v0, s[0:7], s[8:11] dmask:0x1 1310; GFX9-W64-NEXT: ; implicit-def: $vgpr0 1311; GFX9-W64-NEXT: .LBB24_2: ; %Flow 1312; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], s[14:15] 1313; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 1314; GFX9-W64-NEXT: s_and_b64 s[0:1], exec, s[0:1] 1315; GFX9-W64-NEXT: s_xor_b64 exec, exec, s[0:1] 1316; GFX9-W64-NEXT: s_cbranch_execz .LBB24_4 1317; GFX9-W64-NEXT: ; %bb.3: ; %ELSE 1318; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1319; GFX9-W64-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen 1320; GFX9-W64-NEXT: .LBB24_4: ; %END 1321; GFX9-W64-NEXT: s_or_b64 exec, exec, s[0:1] 1322; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1323; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2 1324; GFX9-W64-NEXT: ; return to shader part epilog 1325; 1326; GFX10-W32-LABEL: test_control_flow_1: 1327; GFX10-W32: ; %bb.0: ; %main_body 1328; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo 1329; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1330; GFX10-W32-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1 1331; GFX10-W32-NEXT: s_and_saveexec_b32 s13, vcc_lo 1332; GFX10-W32-NEXT: s_xor_b32 s13, exec_lo, s13 1333; GFX10-W32-NEXT: s_cbranch_execz .LBB24_2 1334; GFX10-W32-NEXT: ; %bb.1: ; %IF 1335; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 1336; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1337; GFX10-W32-NEXT: image_sample v2, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 1338; GFX10-W32-NEXT: ; implicit-def: $vgpr0 1339; GFX10-W32-NEXT: .LBB24_2: ; %Flow 1340; GFX10-W32-NEXT: s_or_saveexec_b32 s0, s13 1341; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 1342; GFX10-W32-NEXT: s_and_b32 s0, exec_lo, s0 1343; GFX10-W32-NEXT: s_xor_b32 exec_lo, exec_lo, s0 1344; GFX10-W32-NEXT: s_cbranch_execz .LBB24_4 1345; GFX10-W32-NEXT: ; %bb.3: ; %ELSE 1346; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1347; GFX10-W32-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen 1348; GFX10-W32-NEXT: .LBB24_4: ; %END 1349; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s0 1350; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1351; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2 1352; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 1353; GFX10-W32-NEXT: ; return to shader part epilog 1354main_body: 1355 %cmp = icmp eq i32 %z, 0 1356 br i1 %cmp, label %ELSE, label %IF 1357 1358IF: 1359 %c.bc = bitcast i32 %c to float 1360 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1361 %tex0 = extractelement <4 x float> %tex, i32 0 1362 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1363 %data.if = extractelement <4 x float> %dtex, i32 0 1364 br label %END 1365 1366ELSE: 1367 call void @llvm.amdgcn.struct.buffer.store.f32(float %data, <4 x i32> undef, i32 %c, i32 0, i32 0, i32 0) 1368 br label %END 1369 1370END: 1371 %r = phi float [ %data.if, %IF ], [ %data, %ELSE ] 1372 ret float %r 1373} 1374 1375; Check that branch conditions are properly marked as needing WQM... 1376define amdgpu_ps <4 x float> @test_control_flow_2(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, <3 x i32> %idx, <2 x float> %data, i32 %coord) { 1377; GFX9-W64-LABEL: test_control_flow_2: 1378; GFX9-W64: ; %bb.0: ; %main_body 1379; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec 1380; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1381; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 1382; GFX9-W64-NEXT: buffer_store_dword v3, v0, s[0:3], 0 idxen 1383; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1384; GFX9-W64-NEXT: buffer_load_dword v0, v1, s[0:3], 0 idxen 1385; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 1386; GFX9-W64-NEXT: buffer_store_dword v4, v2, s[0:3], 0 idxen 1387; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1388; GFX9-W64-NEXT: s_waitcnt vmcnt(1) 1389; GFX9-W64-NEXT: v_cmp_nlt_f32_e32 vcc, 0, v0 1390; GFX9-W64-NEXT: ; implicit-def: $vgpr0 1391; GFX9-W64-NEXT: s_and_saveexec_b64 s[14:15], vcc 1392; GFX9-W64-NEXT: s_xor_b64 s[14:15], exec, s[14:15] 1393; GFX9-W64-NEXT: ; %bb.1: ; %ELSE 1394; GFX9-W64-NEXT: v_lshlrev_b32_e32 v0, 2, v5 1395; GFX9-W64-NEXT: ; implicit-def: $vgpr5 1396; GFX9-W64-NEXT: ; %bb.2: ; %Flow 1397; GFX9-W64-NEXT: s_or_saveexec_b64 s[14:15], s[14:15] 1398; GFX9-W64-NEXT: s_xor_b64 exec, exec, s[14:15] 1399; GFX9-W64-NEXT: ; %bb.3: ; %IF 1400; GFX9-W64-NEXT: v_mul_lo_u32 v0, v5, 3 1401; GFX9-W64-NEXT: ; %bb.4: ; %END 1402; GFX9-W64-NEXT: s_or_b64 exec, exec, s[14:15] 1403; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 1404; GFX9-W64-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf 1405; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1406; GFX9-W64-NEXT: ; return to shader part epilog 1407; 1408; GFX10-W32-LABEL: test_control_flow_2: 1409; GFX10-W32: ; %bb.0: ; %main_body 1410; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo 1411; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1412; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 1413; GFX10-W32-NEXT: buffer_store_dword v3, v0, s[0:3], 0 idxen 1414; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1415; GFX10-W32-NEXT: buffer_load_dword v0, v1, s[0:3], 0 idxen 1416; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1417; GFX10-W32-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0, v0 1418; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 1419; GFX10-W32-NEXT: buffer_store_dword v4, v2, s[0:3], 0 idxen 1420; GFX10-W32-NEXT: ; implicit-def: $vgpr0 1421; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1422; GFX10-W32-NEXT: s_and_saveexec_b32 s13, vcc_lo 1423; GFX10-W32-NEXT: s_xor_b32 s13, exec_lo, s13 1424; GFX10-W32-NEXT: ; %bb.1: ; %ELSE 1425; GFX10-W32-NEXT: v_lshlrev_b32_e32 v0, 2, v5 1426; GFX10-W32-NEXT: ; implicit-def: $vgpr5 1427; GFX10-W32-NEXT: ; %bb.2: ; %Flow 1428; GFX10-W32-NEXT: s_or_saveexec_b32 s13, s13 1429; GFX10-W32-NEXT: s_xor_b32 exec_lo, exec_lo, s13 1430; GFX10-W32-NEXT: ; %bb.3: ; %IF 1431; GFX10-W32-NEXT: v_mul_lo_u32 v0, v5, 3 1432; GFX10-W32-NEXT: ; %bb.4: ; %END 1433; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s13 1434; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 1435; GFX10-W32-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D 1436; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1437; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 1438; GFX10-W32-NEXT: ; return to shader part epilog 1439main_body: 1440 %idx.1 = extractelement <3 x i32> %idx, i32 0 1441 %data.1 = extractelement <2 x float> %data, i32 0 1442 call void @llvm.amdgcn.struct.buffer.store.f32(float %data.1, <4 x i32> undef, i32 %idx.1, i32 0, i32 0, i32 0) 1443 1444 ; The load that determines the branch (and should therefore be WQM) is 1445 ; surrounded by stores that require disabled WQM. 1446 %idx.2 = extractelement <3 x i32> %idx, i32 1 1447 %z = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx.2, i32 0, i32 0, i32 0) 1448 1449 %idx.3 = extractelement <3 x i32> %idx, i32 2 1450 %data.3 = extractelement <2 x float> %data, i32 1 1451 call void @llvm.amdgcn.struct.buffer.store.f32(float %data.3, <4 x i32> undef, i32 %idx.3, i32 0, i32 0, i32 0) 1452 1453 %cc = fcmp ogt float %z, 0.0 1454 br i1 %cc, label %IF, label %ELSE 1455 1456IF: 1457 %coord.IF = mul i32 %coord, 3 1458 br label %END 1459 1460ELSE: 1461 %coord.ELSE = mul i32 %coord, 4 1462 br label %END 1463 1464END: 1465 %coord.END = phi i32 [ %coord.IF, %IF ], [ %coord.ELSE, %ELSE ] 1466 %coord.END.bc = bitcast i32 %coord.END to float 1467 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord.END.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1468 ret <4 x float> %tex 1469} 1470 1471; ... but only if they really do need it. 1472define amdgpu_ps float @test_control_flow_3(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %idx, float %coord) { 1473; GFX9-W64-LABEL: test_control_flow_3: 1474; GFX9-W64: ; %bb.0: ; %main_body 1475; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec 1476; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1477; GFX9-W64-NEXT: image_sample v1, v1, s[0:7], s[8:11] dmask:0x1 1478; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 1479; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1480; GFX9-W64-NEXT: image_sample v1, v1, s[0:7], s[8:11] dmask:0x1 1481; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1482; GFX9-W64-NEXT: v_cmp_nlt_f32_e32 vcc, 0, v1 1483; GFX9-W64-NEXT: buffer_store_dword v1, v0, s[0:3], 0 idxen 1484; GFX9-W64-NEXT: ; implicit-def: $vgpr0 1485; GFX9-W64-NEXT: s_and_saveexec_b64 s[0:1], vcc 1486; GFX9-W64-NEXT: s_xor_b64 s[0:1], exec, s[0:1] 1487; GFX9-W64-NEXT: ; %bb.1: ; %ELSE 1488; GFX9-W64-NEXT: v_mul_f32_e32 v0, 4.0, v1 1489; GFX9-W64-NEXT: ; implicit-def: $vgpr1 1490; GFX9-W64-NEXT: ; %bb.2: ; %Flow 1491; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], s[0:1] 1492; GFX9-W64-NEXT: s_xor_b64 exec, exec, s[0:1] 1493; GFX9-W64-NEXT: ; %bb.3: ; %IF 1494; GFX9-W64-NEXT: v_mul_f32_e32 v0, 0x40400000, v1 1495; GFX9-W64-NEXT: ; %bb.4: ; %END 1496; GFX9-W64-NEXT: s_or_b64 exec, exec, s[0:1] 1497; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1498; GFX9-W64-NEXT: ; return to shader part epilog 1499; 1500; GFX10-W32-LABEL: test_control_flow_3: 1501; GFX10-W32: ; %bb.0: ; %main_body 1502; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo 1503; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1504; GFX10-W32-NEXT: image_sample v1, v1, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 1505; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 1506; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1507; GFX10-W32-NEXT: image_sample v1, v1, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 1508; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1509; GFX10-W32-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0, v1 1510; GFX10-W32-NEXT: buffer_store_dword v1, v0, s[0:3], 0 idxen 1511; GFX10-W32-NEXT: ; implicit-def: $vgpr0 1512; GFX10-W32-NEXT: s_and_saveexec_b32 s0, vcc_lo 1513; GFX10-W32-NEXT: s_xor_b32 s0, exec_lo, s0 1514; GFX10-W32-NEXT: ; %bb.1: ; %ELSE 1515; GFX10-W32-NEXT: v_mul_f32_e32 v0, 4.0, v1 1516; GFX10-W32-NEXT: ; implicit-def: $vgpr1 1517; GFX10-W32-NEXT: ; %bb.2: ; %Flow 1518; GFX10-W32-NEXT: s_or_saveexec_b32 s0, s0 1519; GFX10-W32-NEXT: s_xor_b32 exec_lo, exec_lo, s0 1520; GFX10-W32-NEXT: ; %bb.3: ; %IF 1521; GFX10-W32-NEXT: v_mul_f32_e32 v0, 0x40400000, v1 1522; GFX10-W32-NEXT: ; %bb.4: ; %END 1523; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s0 1524; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 1525; GFX10-W32-NEXT: ; return to shader part epilog 1526main_body: 1527 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1528 %tex0 = extractelement <4 x float> %tex, i32 0 1529 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1530 %dtex.1 = extractelement <4 x float> %dtex, i32 0 1531 call void @llvm.amdgcn.struct.buffer.store.f32(float %dtex.1, <4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 1532 1533 %cc = fcmp ogt float %dtex.1, 0.0 1534 br i1 %cc, label %IF, label %ELSE 1535 1536IF: 1537 %tex.IF = fmul float %dtex.1, 3.0 1538 br label %END 1539 1540ELSE: 1541 %tex.ELSE = fmul float %dtex.1, 4.0 1542 br label %END 1543 1544END: 1545 %tex.END = phi float [ %tex.IF, %IF ], [ %tex.ELSE, %ELSE ] 1546 ret float %tex.END 1547} 1548 1549; Another test that failed at some point because of terminator handling. 1550define amdgpu_ps <4 x float> @test_control_flow_4(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float %coord, i32 %y, float %z) { 1551; GFX9-W64-LABEL: test_control_flow_4: 1552; GFX9-W64: ; %bb.0: ; %main_body 1553; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec 1554; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1555; GFX9-W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 1556; GFX9-W64-NEXT: s_and_saveexec_b64 s[14:15], vcc 1557; GFX9-W64-NEXT: s_cbranch_execz .LBB27_2 1558; GFX9-W64-NEXT: ; %bb.1: ; %IF 1559; GFX9-W64-NEXT: s_and_saveexec_b64 s[16:17], s[12:13] 1560; GFX9-W64-NEXT: buffer_load_dword v1, off, s[0:3], 0 1561; GFX9-W64-NEXT: v_mov_b32_e32 v2, 1 1562; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1563; GFX9-W64-NEXT: buffer_store_dword v1, v2, s[0:3], 0 idxen 1564; GFX9-W64-NEXT: s_mov_b64 exec, s[16:17] 1565; GFX9-W64-NEXT: .LBB27_2: ; %END 1566; GFX9-W64-NEXT: s_or_b64 exec, exec, s[14:15] 1567; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 1568; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 1569; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1570; GFX9-W64-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf 1571; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1572; GFX9-W64-NEXT: ; return to shader part epilog 1573; 1574; GFX10-W32-LABEL: test_control_flow_4: 1575; GFX10-W32: ; %bb.0: ; %main_body 1576; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo 1577; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1578; GFX10-W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1 1579; GFX10-W32-NEXT: s_and_saveexec_b32 s13, vcc_lo 1580; GFX10-W32-NEXT: s_cbranch_execz .LBB27_2 1581; GFX10-W32-NEXT: ; %bb.1: ; %IF 1582; GFX10-W32-NEXT: s_and_saveexec_b32 s14, s12 1583; GFX10-W32-NEXT: buffer_load_dword v1, off, s[0:3], 0 1584; GFX10-W32-NEXT: v_mov_b32_e32 v2, 1 1585; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1586; GFX10-W32-NEXT: buffer_store_dword v1, v2, s[0:3], 0 idxen 1587; GFX10-W32-NEXT: s_mov_b32 exec_lo, s14 1588; GFX10-W32-NEXT: .LBB27_2: ; %END 1589; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s13 1590; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 1591; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 1592; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1593; GFX10-W32-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D 1594; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1595; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 1596; GFX10-W32-NEXT: ; return to shader part epilog 1597main_body: 1598 %cond = icmp eq i32 %y, 0 1599 br i1 %cond, label %IF, label %END 1600 1601IF: 1602 %data = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> undef, i32 0, i32 0, i32 0) 1603 call void @llvm.amdgcn.struct.buffer.store.f32(float %data, <4 x i32> undef, i32 1, i32 0, i32 0, i32 0) 1604 br label %END 1605 1606END: 1607 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1608 %tex0 = extractelement <4 x float> %tex, i32 0 1609 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1610 ret <4 x float> %dtex 1611} 1612 1613; Kill is performed in WQM mode so that uniform kill behaves correctly ... 1614define amdgpu_ps <4 x float> @test_kill_0(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float addrspace(1)* inreg %ptr, <2 x i32> %idx, <2 x float> %data, float %coord, float %coord2, float %z) { 1615; GFX9-W64-LABEL: test_kill_0: 1616; GFX9-W64: ; %bb.0: ; %main_body 1617; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec 1618; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1619; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 1620; GFX9-W64-NEXT: image_sample v[7:10], v4, s[0:7], s[8:11] dmask:0xf 1621; GFX9-W64-NEXT: s_nop 0 1622; GFX9-W64-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen 1623; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1624; GFX9-W64-NEXT: v_cmp_ngt_f32_e32 vcc, 0, v6 1625; GFX9-W64-NEXT: s_andn2_b64 s[12:13], s[12:13], vcc 1626; GFX9-W64-NEXT: s_cbranch_scc0 .LBB28_2 1627; GFX9-W64-NEXT: ; %bb.1: ; %main_body 1628; GFX9-W64-NEXT: s_andn2_b64 exec, exec, vcc 1629; GFX9-W64-NEXT: image_sample v0, v5, s[0:7], s[8:11] dmask:0x1 1630; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 1631; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1632; GFX9-W64-NEXT: image_sample v[11:14], v0, s[0:7], s[8:11] dmask:0xf 1633; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1634; GFX9-W64-NEXT: v_add_f32_e32 v0, v7, v11 1635; GFX9-W64-NEXT: buffer_store_dword v3, v1, s[0:3], 0 idxen 1636; GFX9-W64-NEXT: v_add_f32_e32 v1, v8, v12 1637; GFX9-W64-NEXT: v_add_f32_e32 v2, v9, v13 1638; GFX9-W64-NEXT: v_add_f32_e32 v3, v10, v14 1639; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1640; GFX9-W64-NEXT: s_branch .LBB28_3 1641; GFX9-W64-NEXT: .LBB28_2: 1642; GFX9-W64-NEXT: s_mov_b64 exec, 0 1643; GFX9-W64-NEXT: exp null off, off, off, off done vm 1644; GFX9-W64-NEXT: s_endpgm 1645; GFX9-W64-NEXT: .LBB28_3: 1646; 1647; GFX10-W32-LABEL: test_kill_0: 1648; GFX10-W32: ; %bb.0: ; %main_body 1649; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo 1650; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1651; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 1652; GFX10-W32-NEXT: image_sample v[7:10], v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D 1653; GFX10-W32-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen 1654; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1655; GFX10-W32-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0, v6 1656; GFX10-W32-NEXT: s_andn2_b32 s12, s12, vcc_lo 1657; GFX10-W32-NEXT: s_cbranch_scc0 .LBB28_2 1658; GFX10-W32-NEXT: ; %bb.1: ; %main_body 1659; GFX10-W32-NEXT: s_andn2_b32 exec_lo, exec_lo, vcc_lo 1660; GFX10-W32-NEXT: image_sample v0, v5, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 1661; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 1662; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1663; GFX10-W32-NEXT: image_sample v[11:14], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D 1664; GFX10-W32-NEXT: buffer_store_dword v3, v1, s[0:3], 0 idxen 1665; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1666; GFX10-W32-NEXT: v_add_f32_e32 v4, v8, v12 1667; GFX10-W32-NEXT: v_add_f32_e32 v5, v10, v14 1668; GFX10-W32-NEXT: v_add_f32_e32 v0, v7, v11 1669; GFX10-W32-NEXT: v_add_f32_e32 v2, v9, v13 1670; GFX10-W32-NEXT: v_mov_b32_e32 v1, v4 1671; GFX10-W32-NEXT: v_mov_b32_e32 v3, v5 1672; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 1673; GFX10-W32-NEXT: s_branch .LBB28_3 1674; GFX10-W32-NEXT: .LBB28_2: 1675; GFX10-W32-NEXT: s_mov_b32 exec_lo, 0 1676; GFX10-W32-NEXT: exp null off, off, off, off done vm 1677; GFX10-W32-NEXT: s_endpgm 1678; GFX10-W32-NEXT: .LBB28_3: 1679main_body: 1680 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1681 %idx.0 = extractelement <2 x i32> %idx, i32 0 1682 %data.0 = extractelement <2 x float> %data, i32 0 1683 call void @llvm.amdgcn.struct.buffer.store.f32(float %data.0, <4 x i32> undef, i32 %idx.0, i32 0, i32 0, i32 0) 1684 1685 %z.cmp = fcmp olt float %z, 0.0 1686 call void @llvm.amdgcn.kill(i1 %z.cmp) 1687 1688 %idx.1 = extractelement <2 x i32> %idx, i32 1 1689 %data.1 = extractelement <2 x float> %data, i32 1 1690 call void @llvm.amdgcn.struct.buffer.store.f32(float %data.1, <4 x i32> undef, i32 %idx.1, i32 0, i32 0, i32 0) 1691 %tex2 = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord2, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1692 %tex2.0 = extractelement <4 x float> %tex2, i32 0 1693 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex2.0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1694 %out = fadd <4 x float> %tex, %dtex 1695 1696 ret <4 x float> %out 1697} 1698 1699; ... but only if WQM is necessary. 1700; CHECK-LABEL: {{^}}test_kill_1: 1701; CHECK-NEXT: ; %main_body 1702; CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 1703; CHECK: s_wqm_b64 exec, exec 1704; CHECK: image_sample 1705; CHECK: s_and_b64 exec, exec, [[ORIG]] 1706; CHECK: image_sample 1707; CHECK-NOT: wqm 1708; CHECK-DAG: buffer_store_dword 1709; CHECK-DAG: v_cmp_ 1710define amdgpu_ps <4 x float> @test_kill_1(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %idx, float %data, float %coord, float %coord2, float %z) { 1711; GFX9-W64-LABEL: test_kill_1: 1712; GFX9-W64: ; %bb.0: ; %main_body 1713; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec 1714; GFX9-W64-NEXT: v_mov_b32_e32 v4, v2 1715; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1716; GFX9-W64-NEXT: v_mov_b32_e32 v5, v0 1717; GFX9-W64-NEXT: image_sample v0, v1, s[0:7], s[8:11] dmask:0x1 1718; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 1719; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1720; GFX9-W64-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf 1721; GFX9-W64-NEXT: v_cmp_ngt_f32_e32 vcc, 0, v4 1722; GFX9-W64-NEXT: s_andn2_b64 s[12:13], s[12:13], vcc 1723; GFX9-W64-NEXT: buffer_store_dword v5, off, s[0:3], 0 1724; GFX9-W64-NEXT: s_cbranch_scc0 .LBB29_2 1725; GFX9-W64-NEXT: ; %bb.1: ; %main_body 1726; GFX9-W64-NEXT: s_andn2_b64 exec, exec, vcc 1727; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1728; GFX9-W64-NEXT: s_branch .LBB29_3 1729; GFX9-W64-NEXT: .LBB29_2: 1730; GFX9-W64-NEXT: s_mov_b64 exec, 0 1731; GFX9-W64-NEXT: exp null off, off, off, off done vm 1732; GFX9-W64-NEXT: s_endpgm 1733; GFX9-W64-NEXT: .LBB29_3: 1734; 1735; GFX10-W32-LABEL: test_kill_1: 1736; GFX10-W32: ; %bb.0: ; %main_body 1737; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo 1738; GFX10-W32-NEXT: v_mov_b32_e32 v4, v2 1739; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1740; GFX10-W32-NEXT: v_mov_b32_e32 v5, v0 1741; GFX10-W32-NEXT: image_sample v0, v1, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 1742; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 1743; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1744; GFX10-W32-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D 1745; GFX10-W32-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0, v4 1746; GFX10-W32-NEXT: buffer_store_dword v5, off, s[0:3], 0 1747; GFX10-W32-NEXT: s_andn2_b32 s12, s12, vcc_lo 1748; GFX10-W32-NEXT: s_cbranch_scc0 .LBB29_2 1749; GFX10-W32-NEXT: ; %bb.1: ; %main_body 1750; GFX10-W32-NEXT: s_andn2_b32 exec_lo, exec_lo, vcc_lo 1751; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1752; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 1753; GFX10-W32-NEXT: s_branch .LBB29_3 1754; GFX10-W32-NEXT: .LBB29_2: 1755; GFX10-W32-NEXT: s_mov_b32 exec_lo, 0 1756; GFX10-W32-NEXT: exp null off, off, off, off done vm 1757; GFX10-W32-NEXT: s_endpgm 1758; GFX10-W32-NEXT: .LBB29_3: 1759main_body: 1760 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1761 %tex0 = extractelement <4 x float> %tex, i32 0 1762 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1763 1764 call void @llvm.amdgcn.raw.buffer.store.f32(float %data, <4 x i32> undef, i32 0, i32 0, i32 0) 1765 1766 %z.cmp = fcmp olt float %z, 0.0 1767 call void @llvm.amdgcn.kill(i1 %z.cmp) 1768 1769 ret <4 x float> %dtex 1770} 1771 1772; Check prolog shaders. 1773; CHECK-LABEL: {{^}}test_prolog_1: 1774; CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 1775; CHECK: s_wqm_b64 exec, exec 1776; CHECK: v_add_f32_e32 v0, 1777; CHECK: s_and_b64 exec, exec, [[ORIG]] 1778define amdgpu_ps float @test_prolog_1(float %a, float %b) #5 { 1779; GFX9-W64-LABEL: test_prolog_1: 1780; GFX9-W64: ; %bb.0: ; %main_body 1781; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec 1782; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1783; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v1 1784; GFX9-W64-NEXT: s_and_b64 exec, exec, s[0:1] 1785; GFX9-W64-NEXT: ; return to shader part epilog 1786; 1787; GFX10-W32-LABEL: test_prolog_1: 1788; GFX10-W32: ; %bb.0: ; %main_body 1789; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo 1790; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1791; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v1 1792; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s0 1793; GFX10-W32-NEXT: ; return to shader part epilog 1794main_body: 1795 %s = fadd float %a, %b 1796 ret float %s 1797} 1798 1799; CHECK-LABEL: {{^}}test_loop_vcc: 1800; CHECK-NEXT: ; %entry 1801; CHECK-NEXT: s_mov_b64 [[LIVE:s\[[0-9]+:[0-9]+\]]], exec 1802; CHECK: s_wqm_b64 exec, exec 1803; CHECK: v_mov 1804; CHECK: v_mov 1805; CHECK: v_mov 1806; CHECK: v_mov 1807; CHECK: s_and_b64 exec, exec, [[LIVE]] 1808; CHECK: image_store 1809; CHECK: s_wqm_b64 exec, exec 1810; CHECK-DAG: v_mov_b32_e32 [[CTR:v[0-9]+]], 0 1811; CHECK-DAG: s_mov_b32 [[SEVEN:s[0-9]+]], 0x40e00000 1812 1813; CHECK: [[LOOPHDR:.LBB[0-9]+_[0-9]+]]: ; %body 1814; CHECK: v_add_f32_e32 [[CTR]], 2.0, [[CTR]] 1815; CHECK: [[LOOP:.LBB[0-9]+_[0-9]+]]: ; %loop 1816; CHECK: v_cmp_lt_f32_e32 vcc, [[SEVEN]], [[CTR]] 1817; CHECK: s_cbranch_vccz [[LOOPHDR]] 1818 1819; CHECK: ; %break 1820; CHECK: ; return 1821define amdgpu_ps <4 x float> @test_loop_vcc(<4 x float> %in) nounwind { 1822; GFX9-W64-LABEL: test_loop_vcc: 1823; GFX9-W64: ; %bb.0: ; %entry 1824; GFX9-W64-NEXT: s_mov_b64 s[8:9], exec 1825; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1826; GFX9-W64-NEXT: v_mov_b32_e32 v7, v3 1827; GFX9-W64-NEXT: v_mov_b32_e32 v6, v2 1828; GFX9-W64-NEXT: v_mov_b32_e32 v5, v1 1829; GFX9-W64-NEXT: v_mov_b32_e32 v4, v0 1830; GFX9-W64-NEXT: s_and_b64 exec, exec, s[8:9] 1831; GFX9-W64-NEXT: s_mov_b32 s0, 0 1832; GFX9-W64-NEXT: s_mov_b32 s1, s0 1833; GFX9-W64-NEXT: s_mov_b32 s2, s0 1834; GFX9-W64-NEXT: s_mov_b32 s3, s0 1835; GFX9-W64-NEXT: s_mov_b32 s4, s0 1836; GFX9-W64-NEXT: s_mov_b32 s5, s0 1837; GFX9-W64-NEXT: s_mov_b32 s6, s0 1838; GFX9-W64-NEXT: s_mov_b32 s7, s0 1839; GFX9-W64-NEXT: image_store v[4:7], v0, s[0:7] dmask:0xf unorm 1840; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1841; GFX9-W64-NEXT: v_mov_b32_e32 v8, 0 1842; GFX9-W64-NEXT: s_mov_b32 s10, 0x40e00000 1843; GFX9-W64-NEXT: s_branch .LBB31_2 1844; GFX9-W64-NEXT: .LBB31_1: ; %body 1845; GFX9-W64-NEXT: ; in Loop: Header=BB31_2 Depth=1 1846; GFX9-W64-NEXT: s_mov_b32 s1, s0 1847; GFX9-W64-NEXT: s_mov_b32 s2, s0 1848; GFX9-W64-NEXT: s_mov_b32 s3, s0 1849; GFX9-W64-NEXT: s_mov_b32 s4, s0 1850; GFX9-W64-NEXT: s_mov_b32 s5, s0 1851; GFX9-W64-NEXT: s_mov_b32 s6, s0 1852; GFX9-W64-NEXT: s_mov_b32 s7, s0 1853; GFX9-W64-NEXT: image_sample v[4:7], v0, s[0:7], s[0:3] dmask:0xf 1854; GFX9-W64-NEXT: v_add_f32_e32 v8, 2.0, v8 1855; GFX9-W64-NEXT: s_mov_b64 s[2:3], 0 1856; GFX9-W64-NEXT: s_cbranch_execz .LBB31_4 1857; GFX9-W64-NEXT: .LBB31_2: ; %loop 1858; GFX9-W64-NEXT: ; =>This Inner Loop Header: Depth=1 1859; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1860; GFX9-W64-NEXT: v_mov_b32_e32 v0, v4 1861; GFX9-W64-NEXT: v_cmp_lt_f32_e32 vcc, s10, v8 1862; GFX9-W64-NEXT: v_mov_b32_e32 v1, v5 1863; GFX9-W64-NEXT: v_mov_b32_e32 v2, v6 1864; GFX9-W64-NEXT: v_mov_b32_e32 v3, v7 1865; GFX9-W64-NEXT: s_and_b64 vcc, exec, vcc 1866; GFX9-W64-NEXT: s_cbranch_vccz .LBB31_1 1867; GFX9-W64-NEXT: ; %bb.3: 1868; GFX9-W64-NEXT: s_mov_b64 s[2:3], -1 1869; GFX9-W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7 1870; GFX9-W64-NEXT: ; implicit-def: $vgpr8 1871; GFX9-W64-NEXT: .LBB31_4: ; %break 1872; GFX9-W64-NEXT: s_and_b64 exec, exec, s[8:9] 1873; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1874; GFX9-W64-NEXT: ; return to shader part epilog 1875; 1876; GFX10-W32-LABEL: test_loop_vcc: 1877; GFX10-W32: ; %bb.0: ; %entry 1878; GFX10-W32-NEXT: s_mov_b32 s8, exec_lo 1879; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1880; GFX10-W32-NEXT: v_mov_b32_e32 v8, 0 1881; GFX10-W32-NEXT: s_mov_b32 s0, 0 1882; GFX10-W32-NEXT: s_mov_b32 s1, s0 1883; GFX10-W32-NEXT: s_mov_b32 s2, s0 1884; GFX10-W32-NEXT: s_mov_b32 s3, s0 1885; GFX10-W32-NEXT: s_mov_b32 s4, s0 1886; GFX10-W32-NEXT: s_mov_b32 s5, s0 1887; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s8 1888; GFX10-W32-NEXT: s_mov_b32 s6, s0 1889; GFX10-W32-NEXT: s_mov_b32 s7, s0 1890; GFX10-W32-NEXT: image_store v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm 1891; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1892; GFX10-W32-NEXT: s_branch .LBB31_2 1893; GFX10-W32-NEXT: .p2align 6 1894; GFX10-W32-NEXT: .LBB31_1: ; %body 1895; GFX10-W32-NEXT: ; in Loop: Header=BB31_2 Depth=1 1896; GFX10-W32-NEXT: s_mov_b32 s1, s0 1897; GFX10-W32-NEXT: s_mov_b32 s2, s0 1898; GFX10-W32-NEXT: s_mov_b32 s3, s0 1899; GFX10-W32-NEXT: s_mov_b32 s4, s0 1900; GFX10-W32-NEXT: s_mov_b32 s5, s0 1901; GFX10-W32-NEXT: s_mov_b32 s6, s0 1902; GFX10-W32-NEXT: s_mov_b32 s7, s0 1903; GFX10-W32-NEXT: v_add_f32_e32 v8, 2.0, v8 1904; GFX10-W32-NEXT: image_sample v[0:3], v4, s[0:7], s[0:3] dmask:0xf dim:SQ_RSRC_IMG_1D 1905; GFX10-W32-NEXT: s_mov_b32 s1, 0 1906; GFX10-W32-NEXT: s_cbranch_execz .LBB31_4 1907; GFX10-W32-NEXT: .LBB31_2: ; %loop 1908; GFX10-W32-NEXT: ; =>This Inner Loop Header: Depth=1 1909; GFX10-W32-NEXT: v_cmp_lt_f32_e32 vcc_lo, 0x40e00000, v8 1910; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1911; GFX10-W32-NEXT: v_mov_b32_e32 v7, v3 1912; GFX10-W32-NEXT: v_mov_b32_e32 v6, v2 1913; GFX10-W32-NEXT: v_mov_b32_e32 v5, v1 1914; GFX10-W32-NEXT: v_mov_b32_e32 v4, v0 1915; GFX10-W32-NEXT: s_and_b32 vcc_lo, exec_lo, vcc_lo 1916; GFX10-W32-NEXT: s_cbranch_vccz .LBB31_1 1917; GFX10-W32-NEXT: ; %bb.3: 1918; GFX10-W32-NEXT: s_mov_b32 s1, -1 1919; GFX10-W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3 1920; GFX10-W32-NEXT: ; implicit-def: $vgpr8 1921; GFX10-W32-NEXT: .LBB31_4: ; %break 1922; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s8 1923; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1924; GFX10-W32-NEXT: v_mov_b32_e32 v0, v4 1925; GFX10-W32-NEXT: v_mov_b32_e32 v1, v5 1926; GFX10-W32-NEXT: v_mov_b32_e32 v2, v6 1927; GFX10-W32-NEXT: v_mov_b32_e32 v3, v7 1928; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 1929; GFX10-W32-NEXT: ; return to shader part epilog 1930entry: 1931 call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> %in, i32 15, i32 undef, <8 x i32> undef, i32 0, i32 0) 1932 br label %loop 1933 1934loop: 1935 %ctr.iv = phi float [ 0.0, %entry ], [ %ctr.next, %body ] 1936 %c.iv = phi <4 x float> [ %in, %entry ], [ %c.next, %body ] 1937 %cc = fcmp ogt float %ctr.iv, 7.0 1938 br i1 %cc, label %break, label %body 1939 1940body: 1941 %c.iv0 = extractelement <4 x float> %c.iv, i32 0 1942 %c.next = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.iv0, <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0) #0 1943 %ctr.next = fadd float %ctr.iv, 2.0 1944 br label %loop 1945 1946break: 1947 ret <4 x float> %c.iv 1948} 1949 1950; Only intrinsic stores need exact execution -- other stores do not have 1951; externally visible effects and may require WQM for correctness. 1952; CHECK-LABEL: {{^}}test_alloca: 1953; CHECK: s_mov_b64 [[LIVE:s\[[0-9]+:[0-9]+\]]], exec 1954; CHECK: s_wqm_b64 exec, exec 1955 1956; CHECK: s_and_b64 exec, exec, [[LIVE]] 1957; CHECK: buffer_store_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 1958; CHECK: s_wqm_b64 exec, exec 1959; CHECK: buffer_store_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:4{{$}} 1960; CHECK: s_and_b64 exec, exec, [[LIVE]] 1961; CHECK: buffer_store_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 idxen 1962; CHECK: s_wqm_b64 exec, exec 1963; CHECK: buffer_load_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 offen 1964 1965; CHECK: s_and_b64 exec, exec, [[LIVE]] 1966; CHECK: image_sample 1967; CHECK: buffer_store_dwordx4 1968define amdgpu_ps void @test_alloca(float %data, i32 %a, i32 %idx) nounwind { 1969; GFX9-W64-LABEL: test_alloca: 1970; GFX9-W64: ; %bb.0: ; %entry 1971; GFX9-W64-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 1972; GFX9-W64-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 1973; GFX9-W64-NEXT: s_mov_b32 s10, -1 1974; GFX9-W64-NEXT: s_mov_b32 s11, 0xe00000 1975; GFX9-W64-NEXT: s_add_u32 s8, s8, s0 1976; GFX9-W64-NEXT: s_addc_u32 s9, s9, 0 1977; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec 1978; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1979; GFX9-W64-NEXT: s_and_b64 exec, exec, s[0:1] 1980; GFX9-W64-NEXT: buffer_store_dword v0, off, s[0:3], 0 1981; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1982; GFX9-W64-NEXT: buffer_store_dword v1, off, s[8:11], 0 offset:4 1983; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1984; GFX9-W64-NEXT: s_and_b64 exec, exec, s[0:1] 1985; GFX9-W64-NEXT: v_mov_b32_e32 v1, 1 1986; GFX9-W64-NEXT: buffer_store_dword v0, v1, s[0:3], 0 idxen 1987; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1988; GFX9-W64-NEXT: v_mov_b32_e32 v0, 4 1989; GFX9-W64-NEXT: v_lshl_add_u32 v0, v2, 2, v0 1990; GFX9-W64-NEXT: buffer_load_dword v0, v0, s[8:11], 0 offen 1991; GFX9-W64-NEXT: s_and_b64 exec, exec, s[0:1] 1992; GFX9-W64-NEXT: s_mov_b32 s0, 0 1993; GFX9-W64-NEXT: s_mov_b32 s1, s0 1994; GFX9-W64-NEXT: s_mov_b32 s2, s0 1995; GFX9-W64-NEXT: s_mov_b32 s3, s0 1996; GFX9-W64-NEXT: s_mov_b32 s4, s0 1997; GFX9-W64-NEXT: s_mov_b32 s5, s0 1998; GFX9-W64-NEXT: s_mov_b32 s6, s0 1999; GFX9-W64-NEXT: s_mov_b32 s7, s0 2000; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2001; GFX9-W64-NEXT: image_sample v[0:3], v0, s[0:7], s[0:3] dmask:0xf 2002; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2003; GFX9-W64-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 2004; GFX9-W64-NEXT: s_endpgm 2005; 2006; GFX10-W32-LABEL: test_alloca: 2007; GFX10-W32: ; %bb.0: ; %entry 2008; GFX10-W32-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 2009; GFX10-W32-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 2010; GFX10-W32-NEXT: s_mov_b32 s10, -1 2011; GFX10-W32-NEXT: s_mov_b32 s11, 0x31c16000 2012; GFX10-W32-NEXT: s_add_u32 s8, s8, s0 2013; GFX10-W32-NEXT: s_addc_u32 s9, s9, 0 2014; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo 2015; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 2016; GFX10-W32-NEXT: v_mov_b32_e32 v3, 1 2017; GFX10-W32-NEXT: v_lshl_add_u32 v2, v2, 2, 4 2018; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s0 2019; GFX10-W32-NEXT: buffer_store_dword v0, off, s[0:3], 0 2020; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 2021; GFX10-W32-NEXT: buffer_store_dword v1, off, s[8:11], 0 offset:4 2022; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 2023; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s0 2024; GFX10-W32-NEXT: buffer_store_dword v0, v3, s[0:3], 0 idxen 2025; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 2026; GFX10-W32-NEXT: buffer_load_dword v0, v2, s[8:11], 0 offen 2027; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s0 2028; GFX10-W32-NEXT: s_mov_b32 s0, 0 2029; GFX10-W32-NEXT: s_mov_b32 s1, s0 2030; GFX10-W32-NEXT: s_mov_b32 s2, s0 2031; GFX10-W32-NEXT: s_mov_b32 s3, s0 2032; GFX10-W32-NEXT: s_mov_b32 s4, s0 2033; GFX10-W32-NEXT: s_mov_b32 s5, s0 2034; GFX10-W32-NEXT: s_mov_b32 s6, s0 2035; GFX10-W32-NEXT: s_mov_b32 s7, s0 2036; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2037; GFX10-W32-NEXT: image_sample v[0:3], v0, s[0:7], s[0:3] dmask:0xf dim:SQ_RSRC_IMG_1D 2038; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2039; GFX10-W32-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 2040; GFX10-W32-NEXT: s_endpgm 2041entry: 2042 %array = alloca [32 x i32], align 4, addrspace(5) 2043 2044 call void @llvm.amdgcn.raw.buffer.store.f32(float %data, <4 x i32> undef, i32 0, i32 0, i32 0) 2045 2046 %s.gep = getelementptr [32 x i32], [32 x i32] addrspace(5)* %array, i32 0, i32 0 2047 store volatile i32 %a, i32 addrspace(5)* %s.gep, align 4 2048 2049 call void @llvm.amdgcn.struct.buffer.store.f32(float %data, <4 x i32> undef, i32 1, i32 0, i32 0, i32 0) 2050 2051 %c.gep = getelementptr [32 x i32], [32 x i32] addrspace(5)* %array, i32 0, i32 %idx 2052 %c = load i32, i32 addrspace(5)* %c.gep, align 4 2053 %c.bc = bitcast i32 %c to float 2054 %t = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0) #0 2055 call void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float> %t, <4 x i32> undef, i32 0, i32 0, i32 0) 2056 2057 ret void 2058} 2059 2060; Must return to exact at the end of a non-void returning shader, 2061; otherwise the EXEC mask exported by the epilog will be wrong. This is true 2062; even if the shader has no kills, because a kill could have happened in a 2063; previous shader fragment. 2064; CHECK-LABEL: {{^}}test_nonvoid_return: 2065; CHECK: s_mov_b64 [[LIVE:s\[[0-9]+:[0-9]+\]]], exec 2066; CHECK: s_wqm_b64 exec, exec 2067; CHECK: s_and_b64 exec, exec, [[LIVE]] 2068; CHECK-NOT: exec 2069define amdgpu_ps <4 x float> @test_nonvoid_return() nounwind { 2070; GFX9-W64-LABEL: test_nonvoid_return: 2071; GFX9-W64: ; %bb.0: 2072; GFX9-W64-NEXT: s_mov_b32 s0, 0 2073; GFX9-W64-NEXT: s_mov_b64 s[8:9], exec 2074; GFX9-W64-NEXT: s_mov_b32 s1, s0 2075; GFX9-W64-NEXT: s_mov_b32 s2, s0 2076; GFX9-W64-NEXT: s_mov_b32 s3, s0 2077; GFX9-W64-NEXT: s_mov_b32 s4, s0 2078; GFX9-W64-NEXT: s_mov_b32 s5, s0 2079; GFX9-W64-NEXT: s_mov_b32 s6, s0 2080; GFX9-W64-NEXT: s_mov_b32 s7, s0 2081; GFX9-W64-NEXT: s_wqm_b64 exec, exec 2082; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[0:3] dmask:0x1 2083; GFX9-W64-NEXT: s_and_b64 exec, exec, s[8:9] 2084; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2085; GFX9-W64-NEXT: image_sample v[0:3], v0, s[0:7], s[0:3] dmask:0xf 2086; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2087; GFX9-W64-NEXT: ; return to shader part epilog 2088; 2089; GFX10-W32-LABEL: test_nonvoid_return: 2090; GFX10-W32: ; %bb.0: 2091; GFX10-W32-NEXT: s_mov_b32 s0, 0 2092; GFX10-W32-NEXT: s_mov_b32 s8, exec_lo 2093; GFX10-W32-NEXT: s_mov_b32 s1, s0 2094; GFX10-W32-NEXT: s_mov_b32 s2, s0 2095; GFX10-W32-NEXT: s_mov_b32 s3, s0 2096; GFX10-W32-NEXT: s_mov_b32 s4, s0 2097; GFX10-W32-NEXT: s_mov_b32 s5, s0 2098; GFX10-W32-NEXT: s_mov_b32 s6, s0 2099; GFX10-W32-NEXT: s_mov_b32 s7, s0 2100; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 2101; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[0:3] dmask:0x1 dim:SQ_RSRC_IMG_1D 2102; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s8 2103; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2104; GFX10-W32-NEXT: image_sample v[0:3], v0, s[0:7], s[0:3] dmask:0xf dim:SQ_RSRC_IMG_1D 2105; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2106; GFX10-W32-NEXT: ; return to shader part epilog 2107 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float undef, <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0) #0 2108 %tex0 = extractelement <4 x float> %tex, i32 0 2109 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0) #0 2110 ret <4 x float> %dtex 2111} 2112 2113; CHECK-LABEL: {{^}}test_nonvoid_return_unreachable: 2114; CHECK: s_mov_b64 [[LIVE:s\[[0-9]+:[0-9]+\]]], exec 2115; CHECK: s_wqm_b64 exec, exec 2116; CHECK: s_and_b64 exec, exec, [[LIVE]] 2117; CHECK-NOT: exec 2118define amdgpu_ps <4 x float> @test_nonvoid_return_unreachable(i32 inreg %c) nounwind { 2119; GFX9-W64-LABEL: test_nonvoid_return_unreachable: 2120; GFX9-W64: ; %bb.0: ; %entry 2121; GFX9-W64-NEXT: s_mov_b32 s4, 0 2122; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec 2123; GFX9-W64-NEXT: s_mov_b32 s5, s4 2124; GFX9-W64-NEXT: s_mov_b32 s6, s4 2125; GFX9-W64-NEXT: s_mov_b32 s7, s4 2126; GFX9-W64-NEXT: s_mov_b32 s8, s4 2127; GFX9-W64-NEXT: s_mov_b32 s9, s4 2128; GFX9-W64-NEXT: s_mov_b32 s10, s4 2129; GFX9-W64-NEXT: s_mov_b32 s11, s4 2130; GFX9-W64-NEXT: s_wqm_b64 exec, exec 2131; GFX9-W64-NEXT: image_sample v0, v0, s[4:11], s[0:3] dmask:0x1 2132; GFX9-W64-NEXT: s_and_b64 exec, exec, s[2:3] 2133; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2134; GFX9-W64-NEXT: image_sample v[0:3], v0, s[4:11], s[0:3] dmask:0xf 2135; GFX9-W64-NEXT: s_cmp_lt_i32 s0, 1 2136; GFX9-W64-NEXT: s_cbranch_scc0 .LBB34_2 2137; GFX9-W64-NEXT: ; %bb.1: ; %else 2138; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2139; GFX9-W64-NEXT: s_branch .LBB34_3 2140; GFX9-W64-NEXT: .LBB34_2: ; %if 2141; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2142; GFX9-W64-NEXT: global_store_dwordx4 v[0:1], v[0:3], off 2143; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2144; GFX9-W64-NEXT: .LBB34_3: 2145; 2146; GFX10-W32-LABEL: test_nonvoid_return_unreachable: 2147; GFX10-W32: ; %bb.0: ; %entry 2148; GFX10-W32-NEXT: s_mov_b32 s4, 0 2149; GFX10-W32-NEXT: s_mov_b32 s1, exec_lo 2150; GFX10-W32-NEXT: s_mov_b32 s5, s4 2151; GFX10-W32-NEXT: s_mov_b32 s6, s4 2152; GFX10-W32-NEXT: s_mov_b32 s7, s4 2153; GFX10-W32-NEXT: s_mov_b32 s8, s4 2154; GFX10-W32-NEXT: s_mov_b32 s9, s4 2155; GFX10-W32-NEXT: s_mov_b32 s10, s4 2156; GFX10-W32-NEXT: s_mov_b32 s11, s4 2157; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 2158; GFX10-W32-NEXT: image_sample v0, v0, s[4:11], s[0:3] dmask:0x1 dim:SQ_RSRC_IMG_1D 2159; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s1 2160; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2161; GFX10-W32-NEXT: image_sample v[0:3], v0, s[4:11], s[0:3] dmask:0xf dim:SQ_RSRC_IMG_1D 2162; GFX10-W32-NEXT: s_cmp_lt_i32 s0, 1 2163; GFX10-W32-NEXT: s_cbranch_scc0 .LBB34_2 2164; GFX10-W32-NEXT: ; %bb.1: ; %else 2165; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2166; GFX10-W32-NEXT: s_branch .LBB34_3 2167; GFX10-W32-NEXT: .LBB34_2: ; %if 2168; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2169; GFX10-W32-NEXT: global_store_dwordx4 v[0:1], v[0:3], off 2170; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 2171; GFX10-W32-NEXT: .LBB34_3: 2172entry: 2173 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float undef, <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0) #0 2174 %tex0 = extractelement <4 x float> %tex, i32 0 2175 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0) #0 2176 %cc = icmp sgt i32 %c, 0 2177 br i1 %cc, label %if, label %else 2178 2179if: 2180 store volatile <4 x float> %dtex, <4 x float> addrspace(1)* undef 2181 unreachable 2182 2183else: 2184 ret <4 x float> %dtex 2185} 2186 2187; Test awareness that s_wqm_b64 clobbers SCC. 2188; CHECK-LABEL: {{^}}test_scc: 2189; CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 2190; CHECK: s_wqm_b64 exec, exec 2191; CHECK: s_cmp_ 2192; CHECK-NEXT: s_cbranch_scc 2193; CHECK: ; %else 2194; CHECK: image_sample 2195; CHECK: ; %if 2196; CHECK: image_sample 2197; CHECK: ; %end 2198; CHECK: s_and_b64 exec, exec, [[ORIG]] 2199define amdgpu_ps <4 x float> @test_scc(i32 inreg %sel, i32 %idx) #1 { 2200; GFX9-W64-LABEL: test_scc: 2201; GFX9-W64: ; %bb.0: ; %main_body 2202; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec 2203; GFX9-W64-NEXT: v_mov_b32_e32 v4, v0 2204; GFX9-W64-NEXT: s_wqm_b64 exec, exec 2205; GFX9-W64-NEXT: s_cmp_lt_i32 s0, 1 2206; GFX9-W64-NEXT: s_cbranch_scc0 .LBB35_2 2207; GFX9-W64-NEXT: ; %bb.1: ; %else 2208; GFX9-W64-NEXT: s_mov_b32 s4, 0 2209; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 2210; GFX9-W64-NEXT: v_mov_b32_e32 v1, 1 2211; GFX9-W64-NEXT: s_mov_b32 s5, s4 2212; GFX9-W64-NEXT: s_mov_b32 s6, s4 2213; GFX9-W64-NEXT: s_mov_b32 s7, s4 2214; GFX9-W64-NEXT: s_mov_b32 s8, s4 2215; GFX9-W64-NEXT: s_mov_b32 s9, s4 2216; GFX9-W64-NEXT: s_mov_b32 s10, s4 2217; GFX9-W64-NEXT: s_mov_b32 s11, s4 2218; GFX9-W64-NEXT: image_sample v[0:3], v[0:1], s[4:11], s[0:3] dmask:0xf 2219; GFX9-W64-NEXT: s_cbranch_execz .LBB35_3 2220; GFX9-W64-NEXT: s_branch .LBB35_4 2221; GFX9-W64-NEXT: .LBB35_2: 2222; GFX9-W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3 2223; GFX9-W64-NEXT: .LBB35_3: ; %if 2224; GFX9-W64-NEXT: s_mov_b32 s4, 0 2225; GFX9-W64-NEXT: s_mov_b32 s5, s4 2226; GFX9-W64-NEXT: s_mov_b32 s6, s4 2227; GFX9-W64-NEXT: s_mov_b32 s7, s4 2228; GFX9-W64-NEXT: s_mov_b32 s8, s4 2229; GFX9-W64-NEXT: s_mov_b32 s9, s4 2230; GFX9-W64-NEXT: s_mov_b32 s10, s4 2231; GFX9-W64-NEXT: s_mov_b32 s11, s4 2232; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2233; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 2234; GFX9-W64-NEXT: image_sample v[0:3], v0, s[4:11], s[0:3] dmask:0xf 2235; GFX9-W64-NEXT: .LBB35_4: ; %end 2236; GFX9-W64-NEXT: s_and_b64 exec, exec, s[2:3] 2237; GFX9-W64-NEXT: v_mov_b32_e32 v5, 1.0 2238; GFX9-W64-NEXT: buffer_store_dword v5, v4, s[0:3], 0 idxen 2239; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2240; GFX9-W64-NEXT: ; return to shader part epilog 2241; 2242; GFX10-W32-LABEL: test_scc: 2243; GFX10-W32: ; %bb.0: ; %main_body 2244; GFX10-W32-NEXT: v_mov_b32_e32 v4, v0 2245; GFX10-W32-NEXT: s_mov_b32 s8, exec_lo 2246; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 2247; GFX10-W32-NEXT: s_cmp_lt_i32 s0, 1 2248; GFX10-W32-NEXT: s_cbranch_scc0 .LBB35_2 2249; GFX10-W32-NEXT: ; %bb.1: ; %else 2250; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 2251; GFX10-W32-NEXT: v_mov_b32_e32 v1, 1 2252; GFX10-W32-NEXT: s_mov_b32 s0, 0 2253; GFX10-W32-NEXT: s_mov_b32 s1, s0 2254; GFX10-W32-NEXT: s_mov_b32 s2, s0 2255; GFX10-W32-NEXT: s_mov_b32 s3, s0 2256; GFX10-W32-NEXT: s_mov_b32 s4, s0 2257; GFX10-W32-NEXT: s_mov_b32 s5, s0 2258; GFX10-W32-NEXT: s_mov_b32 s6, s0 2259; GFX10-W32-NEXT: s_mov_b32 s7, s0 2260; GFX10-W32-NEXT: image_sample v[0:3], v[0:1], s[0:7], s[0:3] dmask:0xf dim:SQ_RSRC_IMG_2D 2261; GFX10-W32-NEXT: s_cbranch_execz .LBB35_3 2262; GFX10-W32-NEXT: s_branch .LBB35_4 2263; GFX10-W32-NEXT: .LBB35_2: 2264; GFX10-W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3 2265; GFX10-W32-NEXT: .LBB35_3: ; %if 2266; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2267; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 2268; GFX10-W32-NEXT: s_mov_b32 s0, 0 2269; GFX10-W32-NEXT: s_mov_b32 s1, s0 2270; GFX10-W32-NEXT: s_mov_b32 s2, s0 2271; GFX10-W32-NEXT: s_mov_b32 s3, s0 2272; GFX10-W32-NEXT: s_mov_b32 s4, s0 2273; GFX10-W32-NEXT: s_mov_b32 s5, s0 2274; GFX10-W32-NEXT: s_mov_b32 s6, s0 2275; GFX10-W32-NEXT: s_mov_b32 s7, s0 2276; GFX10-W32-NEXT: image_sample v[0:3], v0, s[0:7], s[0:3] dmask:0xf dim:SQ_RSRC_IMG_1D 2277; GFX10-W32-NEXT: .LBB35_4: ; %end 2278; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s8 2279; GFX10-W32-NEXT: v_mov_b32_e32 v5, 1.0 2280; GFX10-W32-NEXT: buffer_store_dword v5, v4, s[0:3], 0 idxen 2281; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2282; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 2283; GFX10-W32-NEXT: ; return to shader part epilog 2284main_body: 2285 %cc = icmp sgt i32 %sel, 0 2286 br i1 %cc, label %if, label %else 2287 2288if: 2289 %r.if = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float 0.0, <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0) #0 2290 br label %end 2291 2292else: 2293 %r.else = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32 15, float 0.0, float bitcast (i32 1 to float), <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0) #0 2294 br label %end 2295 2296end: 2297 %r = phi <4 x float> [ %r.if, %if ], [ %r.else, %else ] 2298 call void @llvm.amdgcn.struct.buffer.store.f32(float 1.0, <4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 2299 ret <4 x float> %r 2300} 2301 2302; Check a case of a block being entirely WQM except for a bit of WWM. 2303; There was a bug where it forgot to enter and leave WWM. 2304define amdgpu_ps float @test_wwm_within_wqm(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data) { 2305; GFX9-W64-LABEL: test_wwm_within_wqm: 2306; GFX9-W64: ; %bb.0: ; %main_body 2307; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec 2308; GFX9-W64-NEXT: s_wqm_b64 exec, exec 2309; GFX9-W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 2310; GFX9-W64-NEXT: v_mov_b32_e32 v1, 0 2311; GFX9-W64-NEXT: s_and_saveexec_b64 s[14:15], vcc 2312; GFX9-W64-NEXT: s_cbranch_execz .LBB36_2 2313; GFX9-W64-NEXT: ; %bb.1: ; %IF 2314; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 2315; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2316; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 2317; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2318; GFX9-W64-NEXT: v_cvt_i32_f32_e32 v2, v0 2319; GFX9-W64-NEXT: s_not_b64 exec, exec 2320; GFX9-W64-NEXT: v_mov_b32_e32 v2, 0 2321; GFX9-W64-NEXT: s_not_b64 exec, exec 2322; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], -1 2323; GFX9-W64-NEXT: ds_swizzle_b32 v2, v2 offset:swizzle(SWAP,2) 2324; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1] 2325; GFX9-W64-NEXT: s_waitcnt lgkmcnt(0) 2326; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2 2327; GFX9-W64-NEXT: v_cvt_f32_i32_e32 v1, v0 2328; GFX9-W64-NEXT: .LBB36_2: ; %ENDIF 2329; GFX9-W64-NEXT: s_or_b64 exec, exec, s[14:15] 2330; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 2331; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 2332; GFX9-W64-NEXT: ; return to shader part epilog 2333; 2334; GFX10-W32-LABEL: test_wwm_within_wqm: 2335; GFX10-W32: ; %bb.0: ; %main_body 2336; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo 2337; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 2338; GFX10-W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1 2339; GFX10-W32-NEXT: v_mov_b32_e32 v1, 0 2340; GFX10-W32-NEXT: s_and_saveexec_b32 s13, vcc_lo 2341; GFX10-W32-NEXT: s_cbranch_execz .LBB36_2 2342; GFX10-W32-NEXT: ; %bb.1: ; %IF 2343; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 2344; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2345; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 2346; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2347; GFX10-W32-NEXT: v_cvt_i32_f32_e32 v2, v0 2348; GFX10-W32-NEXT: s_not_b32 exec_lo, exec_lo 2349; GFX10-W32-NEXT: v_mov_b32_e32 v2, 0 2350; GFX10-W32-NEXT: s_not_b32 exec_lo, exec_lo 2351; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 2352; GFX10-W32-NEXT: ds_swizzle_b32 v2, v2 offset:swizzle(SWAP,2) 2353; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 2354; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0) 2355; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2 2356; GFX10-W32-NEXT: v_cvt_f32_i32_e32 v1, v0 2357; GFX10-W32-NEXT: .LBB36_2: ; %ENDIF 2358; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s13 2359; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 2360; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 2361; GFX10-W32-NEXT: ; return to shader part epilog 2362main_body: 2363 %c.bc = bitcast i32 %c to float 2364 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 2365 %tex0 = extractelement <4 x float> %tex, i32 0 2366 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 2367 %cmp = icmp eq i32 %z, 0 2368 br i1 %cmp, label %IF, label %ENDIF 2369 2370IF: 2371 %dataf = extractelement <4 x float> %dtex, i32 0 2372 %data1 = fptosi float %dataf to i32 2373 %data2 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %data1, i32 0) 2374 %data3 = call i32 @llvm.amdgcn.ds.swizzle(i32 %data2, i32 2079) 2375 %data4 = call i32 @llvm.amdgcn.wwm.i32(i32 %data3) 2376 %data4f = sitofp i32 %data4 to float 2377 br label %ENDIF 2378 2379ENDIF: 2380 %r = phi float [ %data4f, %IF ], [ 0.0, %main_body ] 2381 ret float %r 2382} 2383 2384; Check that WWM is triggered by the strict_wwm intrinsic. 2385define amdgpu_ps float @test_strict_wwm1(i32 inreg %idx0, i32 inreg %idx1) { 2386; GFX9-W64-LABEL: test_strict_wwm1: 2387; GFX9-W64: ; %bb.0: ; %main_body 2388; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1 2389; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 2390; GFX9-W64-NEXT: v_mov_b32_e32 v2, s1 2391; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 2392; GFX9-W64-NEXT: s_nop 0 2393; GFX9-W64-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen 2394; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2395; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v2 2396; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 2397; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 2398; GFX9-W64-NEXT: ; return to shader part epilog 2399; 2400; GFX10-W32-LABEL: test_strict_wwm1: 2401; GFX10-W32: ; %bb.0: ; %main_body 2402; GFX10-W32-NEXT: s_or_saveexec_b32 s2, -1 2403; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 2404; GFX10-W32-NEXT: v_mov_b32_e32 v2, s1 2405; GFX10-W32-NEXT: s_clause 0x1 2406; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 2407; GFX10-W32-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen 2408; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2409; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v2 2410; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2 2411; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 2412; GFX10-W32-NEXT: ; return to shader part epilog 2413main_body: 2414 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 2415 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 2416 %out = fadd float %src0, %src1 2417 %out.0 = call float @llvm.amdgcn.strict.wwm.f32(float %out) 2418 ret float %out.0 2419} 2420 2421; Same as above, but with an integer type. 2422define amdgpu_ps float @test_strict_wwm2(i32 inreg %idx0, i32 inreg %idx1) { 2423; GFX9-W64-LABEL: test_strict_wwm2: 2424; GFX9-W64: ; %bb.0: ; %main_body 2425; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1 2426; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 2427; GFX9-W64-NEXT: v_mov_b32_e32 v2, s1 2428; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 2429; GFX9-W64-NEXT: s_nop 0 2430; GFX9-W64-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen 2431; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2432; GFX9-W64-NEXT: v_add_u32_e32 v1, v1, v2 2433; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 2434; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 2435; GFX9-W64-NEXT: ; return to shader part epilog 2436; 2437; GFX10-W32-LABEL: test_strict_wwm2: 2438; GFX10-W32: ; %bb.0: ; %main_body 2439; GFX10-W32-NEXT: s_or_saveexec_b32 s2, -1 2440; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 2441; GFX10-W32-NEXT: v_mov_b32_e32 v2, s1 2442; GFX10-W32-NEXT: s_clause 0x1 2443; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 2444; GFX10-W32-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen 2445; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2446; GFX10-W32-NEXT: v_add_nc_u32_e32 v1, v1, v2 2447; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2 2448; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 2449; GFX10-W32-NEXT: ; return to shader part epilog 2450main_body: 2451 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 2452 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 2453 %src0.0 = bitcast float %src0 to i32 2454 %src1.0 = bitcast float %src1 to i32 2455 %out = add i32 %src0.0, %src1.0 2456 %out.0 = call i32 @llvm.amdgcn.strict.wwm.i32(i32 %out) 2457 %out.1 = bitcast i32 %out.0 to float 2458 ret float %out.1 2459} 2460 2461; Check that we don't leave WWM on for computations that don't require WWM, 2462; since that will lead clobbering things that aren't supposed to be clobbered 2463; in cases like this. 2464; We enforce this by checking that v_add gets emitted in the same block as 2465; WWM computations. 2466define amdgpu_ps float @test_strict_wwm3(i32 inreg %idx) { 2467; GFX9-W64-LABEL: test_strict_wwm3: 2468; GFX9-W64: ; %bb.0: ; %main_body 2469; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 2470; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 2471; GFX9-W64-NEXT: v_cmp_gt_u32_e32 vcc, 16, v0 2472; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 2473; GFX9-W64-NEXT: s_and_saveexec_b64 s[2:3], vcc 2474; GFX9-W64-NEXT: s_cbranch_execz .LBB39_2 2475; GFX9-W64-NEXT: ; %bb.1: ; %if 2476; GFX9-W64-NEXT: s_or_saveexec_b64 s[4:5], -1 2477; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 2478; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 2479; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2480; GFX9-W64-NEXT: v_add_f32_e32 v2, v1, v1 2481; GFX9-W64-NEXT: s_mov_b64 exec, s[4:5] 2482; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2 2483; GFX9-W64-NEXT: v_add_f32_e32 v0, v1, v0 2484; GFX9-W64-NEXT: .LBB39_2: ; %endif 2485; GFX9-W64-NEXT: s_or_b64 exec, exec, s[2:3] 2486; GFX9-W64-NEXT: ; return to shader part epilog 2487; 2488; GFX10-W32-LABEL: test_strict_wwm3: 2489; GFX10-W32: ; %bb.0: ; %main_body 2490; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 2491; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 2492; GFX10-W32-NEXT: v_cmp_gt_u32_e32 vcc_lo, 16, v0 2493; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 2494; GFX10-W32-NEXT: s_and_saveexec_b32 s1, vcc_lo 2495; GFX10-W32-NEXT: s_cbranch_execz .LBB39_2 2496; GFX10-W32-NEXT: ; %bb.1: ; %if 2497; GFX10-W32-NEXT: s_or_saveexec_b32 s2, -1 2498; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 2499; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 2500; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2501; GFX10-W32-NEXT: v_add_f32_e32 v2, v1, v1 2502; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2 2503; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2 2504; GFX10-W32-NEXT: v_add_f32_e32 v0, v1, v0 2505; GFX10-W32-NEXT: .LBB39_2: ; %endif 2506; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s1 2507; GFX10-W32-NEXT: ; return to shader part epilog 2508main_body: 2509 ; use mbcnt to make sure the branch is divergent 2510 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 2511 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 2512 %cc = icmp uge i32 %hi, 16 2513 br i1 %cc, label %endif, label %if 2514 2515if: 2516 %src = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 2517 %out = fadd float %src, %src 2518 %out.0 = call float @llvm.amdgcn.strict.wwm.f32(float %out) 2519 %out.1 = fadd float %src, %out.0 2520 br label %endif 2521 2522endif: 2523 %out.2 = phi float [ %out.1, %if ], [ 0.0, %main_body ] 2524 ret float %out.2 2525} 2526 2527; Check that WWM writes aren't coalesced with non-WWM writes, since the WWM 2528; write could clobber disabled channels in the non-WWM one. 2529; We enforce this by checking that v_mov gets emitted in the same block as 2530; WWM computations. 2531define amdgpu_ps float @test_strict_wwm4(i32 inreg %idx) { 2532; GFX9-W64-LABEL: test_strict_wwm4: 2533; GFX9-W64: ; %bb.0: ; %main_body 2534; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 2535; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 2536; GFX9-W64-NEXT: v_cmp_gt_u32_e32 vcc, 16, v0 2537; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 2538; GFX9-W64-NEXT: s_and_saveexec_b64 s[2:3], vcc 2539; GFX9-W64-NEXT: s_cbranch_execz .LBB40_2 2540; GFX9-W64-NEXT: ; %bb.1: ; %if 2541; GFX9-W64-NEXT: s_or_saveexec_b64 s[4:5], -1 2542; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 2543; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 2544; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2545; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v1 2546; GFX9-W64-NEXT: s_mov_b64 exec, s[4:5] 2547; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 2548; GFX9-W64-NEXT: .LBB40_2: ; %endif 2549; GFX9-W64-NEXT: s_or_b64 exec, exec, s[2:3] 2550; GFX9-W64-NEXT: ; return to shader part epilog 2551; 2552; GFX10-W32-LABEL: test_strict_wwm4: 2553; GFX10-W32: ; %bb.0: ; %main_body 2554; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 2555; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 2556; GFX10-W32-NEXT: v_cmp_gt_u32_e32 vcc_lo, 16, v0 2557; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 2558; GFX10-W32-NEXT: s_and_saveexec_b32 s1, vcc_lo 2559; GFX10-W32-NEXT: s_cbranch_execz .LBB40_2 2560; GFX10-W32-NEXT: ; %bb.1: ; %if 2561; GFX10-W32-NEXT: s_or_saveexec_b32 s2, -1 2562; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 2563; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 2564; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2565; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v1 2566; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2 2567; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 2568; GFX10-W32-NEXT: .LBB40_2: ; %endif 2569; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s1 2570; GFX10-W32-NEXT: ; return to shader part epilog 2571main_body: 2572 ; use mbcnt to make sure the branch is divergent 2573 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 2574 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 2575 %cc = icmp uge i32 %hi, 16 2576 br i1 %cc, label %endif, label %if 2577 2578if: 2579 %src = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 2580 %out = fadd float %src, %src 2581 %out.0 = call float @llvm.amdgcn.strict.wwm.f32(float %out) 2582 br label %endif 2583 2584endif: 2585 %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ] 2586 ret float %out.1 2587} 2588 2589; Make sure the transition from Exact to WWM then WQM works properly. 2590define amdgpu_ps float @test_strict_wwm5(i32 inreg %idx0, i32 inreg %idx1) { 2591; GFX9-W64-LABEL: test_strict_wwm5: 2592; GFX9-W64: ; %bb.0: ; %main_body 2593; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec 2594; GFX9-W64-NEXT: v_mov_b32_e32 v0, s0 2595; GFX9-W64-NEXT: buffer_load_dword v2, v0, s[0:3], 0 idxen 2596; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2597; GFX9-W64-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen 2598; GFX9-W64-NEXT: s_or_saveexec_b64 s[4:5], -1 2599; GFX9-W64-NEXT: v_mov_b32_e32 v1, s1 2600; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 2601; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2602; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v1 2603; GFX9-W64-NEXT: s_mov_b64 exec, s[4:5] 2604; GFX9-W64-NEXT: s_wqm_b64 exec, exec 2605; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 2606; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v0 2607; GFX9-W64-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec killed $exec 2608; GFX9-W64-NEXT: s_and_b64 exec, exec, s[2:3] 2609; GFX9-W64-NEXT: ; return to shader part epilog 2610; 2611; GFX10-W32-LABEL: test_strict_wwm5: 2612; GFX10-W32: ; %bb.0: ; %main_body 2613; GFX10-W32-NEXT: v_mov_b32_e32 v0, s0 2614; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo 2615; GFX10-W32-NEXT: buffer_load_dword v2, v0, s[0:3], 0 idxen 2616; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 2617; GFX10-W32-NEXT: v_mov_b32_e32 v1, s1 2618; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 2619; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2620; GFX10-W32-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen 2621; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 2622; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 2623; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2624; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v1 2625; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 2626; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 2627; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 2628; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v0 2629; GFX10-W32-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec killed $exec 2630; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s2 2631; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 2632; GFX10-W32-NEXT: ; return to shader part epilog 2633main_body: 2634 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 2635 call void @llvm.amdgcn.struct.buffer.store.f32(float %src0, <4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 2636 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 2637 %temp = fadd float %src1, %src1 2638 %temp.0 = call float @llvm.amdgcn.strict.wwm.f32(float %temp) 2639 %out = fadd float %temp.0, %temp.0 2640 %out.0 = call float @llvm.amdgcn.wqm.f32(float %out) 2641 ret float %out.0 2642} 2643 2644; Check that WWM is turned on correctly across basic block boundaries. 2645; if..then..endif version 2646;SI-CHECK: buffer_load_dword 2647;VI-CHECK: flat_load_dword 2648;SI-CHECK: buffer_load_dword 2649;VI-CHECK: flat_load_dword 2650define amdgpu_ps float @test_strict_wwm6_then() { 2651; GFX9-W64-LABEL: test_strict_wwm6_then: 2652; GFX9-W64: ; %bb.0: ; %main_body 2653; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], -1 2654; GFX9-W64-NEXT: global_load_dword v1, v[3:4], off glc 2655; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2656; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1] 2657; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 2658; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 2659; GFX9-W64-NEXT: v_cmp_gt_u32_e32 vcc, 16, v0 2660; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 2661; GFX9-W64-NEXT: s_and_saveexec_b64 s[0:1], vcc 2662; GFX9-W64-NEXT: s_cbranch_execz .LBB42_2 2663; GFX9-W64-NEXT: ; %bb.1: ; %if 2664; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1 2665; GFX9-W64-NEXT: global_load_dword v2, v[3:4], off glc 2666; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2667; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v2 2668; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 2669; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 2670; GFX9-W64-NEXT: .LBB42_2: ; %endif 2671; GFX9-W64-NEXT: s_or_b64 exec, exec, s[0:1] 2672; GFX9-W64-NEXT: ; return to shader part epilog 2673; 2674; GFX10-W32-LABEL: test_strict_wwm6_then: 2675; GFX10-W32: ; %bb.0: ; %main_body 2676; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 2677; GFX10-W32-NEXT: global_load_dword v1, v[3:4], off glc dlc 2678; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2679; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 2680; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 2681; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 2682; GFX10-W32-NEXT: v_cmp_gt_u32_e32 vcc_lo, 16, v0 2683; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 2684; GFX10-W32-NEXT: s_and_saveexec_b32 s0, vcc_lo 2685; GFX10-W32-NEXT: s_cbranch_execz .LBB42_2 2686; GFX10-W32-NEXT: ; %bb.1: ; %if 2687; GFX10-W32-NEXT: s_or_saveexec_b32 s1, -1 2688; GFX10-W32-NEXT: global_load_dword v2, v[3:4], off glc dlc 2689; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2690; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v2 2691; GFX10-W32-NEXT: s_mov_b32 exec_lo, s1 2692; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 2693; GFX10-W32-NEXT: .LBB42_2: ; %endif 2694; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s0 2695; GFX10-W32-NEXT: ; return to shader part epilog 2696main_body: 2697 %src0 = load volatile float, float addrspace(1)* undef 2698 ; use mbcnt to make sure the branch is divergent 2699 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 2700 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 2701 %cc = icmp uge i32 %hi, 16 2702 br i1 %cc, label %endif, label %if 2703 2704if: 2705 %src1 = load volatile float, float addrspace(1)* undef 2706 %out = fadd float %src0, %src1 2707 %out.0 = call float @llvm.amdgcn.strict.wwm.f32(float %out) 2708 br label %endif 2709 2710endif: 2711 %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ] 2712 ret float %out.1 2713} 2714 2715; Check that WWM is turned on correctly across basic block boundaries. 2716; loop version 2717define amdgpu_ps float @test_strict_wwm6_loop() { 2718; GFX9-W64-LABEL: test_strict_wwm6_loop: 2719; GFX9-W64: ; %bb.0: ; %main_body 2720; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], -1 2721; GFX9-W64-NEXT: global_load_dword v1, v[3:4], off glc 2722; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2723; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1] 2724; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 2725; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v0 2726; GFX9-W64-NEXT: s_mov_b64 s[0:1], 0 2727; GFX9-W64-NEXT: .LBB43_1: ; %loop 2728; GFX9-W64-NEXT: ; =>This Inner Loop Header: Depth=1 2729; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1 2730; GFX9-W64-NEXT: global_load_dword v2, v[3:4], off glc 2731; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2732; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 2733; GFX9-W64-NEXT: v_add_u32_e32 v3, -1, v3 2734; GFX9-W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3 2735; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1 2736; GFX9-W64-NEXT: v_add_f32_e32 v2, v1, v2 2737; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 2738; GFX9-W64-NEXT: s_or_b64 s[0:1], vcc, s[0:1] 2739; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2 2740; GFX9-W64-NEXT: s_andn2_b64 exec, exec, s[0:1] 2741; GFX9-W64-NEXT: s_cbranch_execnz .LBB43_1 2742; GFX9-W64-NEXT: ; %bb.2: ; %endloop 2743; GFX9-W64-NEXT: s_or_b64 exec, exec, s[0:1] 2744; GFX9-W64-NEXT: ; return to shader part epilog 2745; 2746; GFX10-W32-LABEL: test_strict_wwm6_loop: 2747; GFX10-W32: ; %bb.0: ; %main_body 2748; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 2749; GFX10-W32-NEXT: global_load_dword v1, v[3:4], off glc dlc 2750; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2751; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 2752; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 2753; GFX10-W32-NEXT: s_mov_b32 s0, 0 2754; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v0 2755; GFX10-W32-NEXT: .LBB43_1: ; %loop 2756; GFX10-W32-NEXT: ; =>This Inner Loop Header: Depth=1 2757; GFX10-W32-NEXT: s_or_saveexec_b32 s1, -1 2758; GFX10-W32-NEXT: global_load_dword v2, v[3:4], off glc dlc 2759; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2760; GFX10-W32-NEXT: s_mov_b32 exec_lo, s1 2761; GFX10-W32-NEXT: v_add_nc_u32_e32 v3, -1, v3 2762; GFX10-W32-NEXT: s_or_saveexec_b32 s1, -1 2763; GFX10-W32-NEXT: v_add_f32_e32 v2, v1, v2 2764; GFX10-W32-NEXT: s_mov_b32 exec_lo, s1 2765; GFX10-W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v3 2766; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2 2767; GFX10-W32-NEXT: s_or_b32 s0, vcc_lo, s0 2768; GFX10-W32-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 2769; GFX10-W32-NEXT: s_cbranch_execnz .LBB43_1 2770; GFX10-W32-NEXT: ; %bb.2: ; %endloop 2771; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s0 2772; GFX10-W32-NEXT: ; return to shader part epilog 2773main_body: 2774 %src0 = load volatile float, float addrspace(1)* undef 2775 ; use mbcnt to make sure the branch is divergent 2776 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 2777 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 2778 br label %loop 2779 2780loop: 2781 %counter = phi i32 [ %hi, %main_body ], [ %counter.1, %loop ] 2782 %src1 = load volatile float, float addrspace(1)* undef 2783 %out = fadd float %src0, %src1 2784 %out.0 = call float @llvm.amdgcn.strict.wwm.f32(float %out) 2785 %counter.1 = sub i32 %counter, 1 2786 %cc = icmp ne i32 %counter.1, 0 2787 br i1 %cc, label %loop, label %endloop 2788 2789endloop: 2790 ret float %out.0 2791} 2792 2793; Check that @llvm.amdgcn.set.inactive disables WWM. 2794define amdgpu_ps void @test_strict_wwm_set_inactive1(i32 inreg %idx) { 2795; GFX9-W64-LABEL: test_strict_wwm_set_inactive1: 2796; GFX9-W64: ; %bb.0: ; %main_body 2797; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 2798; GFX9-W64-NEXT: buffer_load_dword v0, v1, s[0:3], 0 idxen 2799; GFX9-W64-NEXT: s_not_b64 exec, exec 2800; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2801; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 2802; GFX9-W64-NEXT: s_not_b64 exec, exec 2803; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], -1 2804; GFX9-W64-NEXT: v_add_u32_e32 v0, v0, v0 2805; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1] 2806; GFX9-W64-NEXT: v_mov_b32_e32 v2, v0 2807; GFX9-W64-NEXT: buffer_store_dword v2, v1, s[0:3], 0 idxen 2808; GFX9-W64-NEXT: s_endpgm 2809; 2810; GFX10-W32-LABEL: test_strict_wwm_set_inactive1: 2811; GFX10-W32: ; %bb.0: ; %main_body 2812; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 2813; GFX10-W32-NEXT: buffer_load_dword v0, v1, s[0:3], 0 idxen 2814; GFX10-W32-NEXT: s_not_b32 exec_lo, exec_lo 2815; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2816; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 2817; GFX10-W32-NEXT: s_not_b32 exec_lo, exec_lo 2818; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 2819; GFX10-W32-NEXT: v_add_nc_u32_e32 v0, v0, v0 2820; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 2821; GFX10-W32-NEXT: v_mov_b32_e32 v2, v0 2822; GFX10-W32-NEXT: buffer_store_dword v2, v1, s[0:3], 0 idxen 2823; GFX10-W32-NEXT: s_endpgm 2824main_body: 2825 %src = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 2826 %src.0 = bitcast float %src to i32 2827 %src.1 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %src.0, i32 0) 2828 %out = add i32 %src.1, %src.1 2829 %out.0 = call i32 @llvm.amdgcn.strict.wwm.i32(i32 %out) 2830 %out.1 = bitcast i32 %out.0 to float 2831 call void @llvm.amdgcn.struct.buffer.store.f32(float %out.1, <4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 2832 ret void 2833} 2834 2835; Check a case of a block being entirely WQM except for a bit of WWM. 2836; There was a bug where it forgot to enter and leave WWM. 2837define amdgpu_ps float @test_strict_wwm_within_wqm(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data) { 2838; GFX9-W64-LABEL: test_strict_wwm_within_wqm: 2839; GFX9-W64: ; %bb.0: ; %main_body 2840; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec 2841; GFX9-W64-NEXT: s_wqm_b64 exec, exec 2842; GFX9-W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 2843; GFX9-W64-NEXT: v_mov_b32_e32 v1, 0 2844; GFX9-W64-NEXT: s_and_saveexec_b64 s[14:15], vcc 2845; GFX9-W64-NEXT: s_cbranch_execz .LBB45_2 2846; GFX9-W64-NEXT: ; %bb.1: ; %IF 2847; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 2848; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2849; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 2850; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2851; GFX9-W64-NEXT: v_cvt_i32_f32_e32 v2, v0 2852; GFX9-W64-NEXT: s_not_b64 exec, exec 2853; GFX9-W64-NEXT: v_mov_b32_e32 v2, 0 2854; GFX9-W64-NEXT: s_not_b64 exec, exec 2855; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], -1 2856; GFX9-W64-NEXT: ds_swizzle_b32 v2, v2 offset:swizzle(SWAP,2) 2857; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1] 2858; GFX9-W64-NEXT: s_waitcnt lgkmcnt(0) 2859; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2 2860; GFX9-W64-NEXT: v_cvt_f32_i32_e32 v1, v0 2861; GFX9-W64-NEXT: .LBB45_2: ; %ENDIF 2862; GFX9-W64-NEXT: s_or_b64 exec, exec, s[14:15] 2863; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 2864; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 2865; GFX9-W64-NEXT: ; return to shader part epilog 2866; 2867; GFX10-W32-LABEL: test_strict_wwm_within_wqm: 2868; GFX10-W32: ; %bb.0: ; %main_body 2869; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo 2870; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 2871; GFX10-W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1 2872; GFX10-W32-NEXT: v_mov_b32_e32 v1, 0 2873; GFX10-W32-NEXT: s_and_saveexec_b32 s13, vcc_lo 2874; GFX10-W32-NEXT: s_cbranch_execz .LBB45_2 2875; GFX10-W32-NEXT: ; %bb.1: ; %IF 2876; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 2877; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2878; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 2879; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2880; GFX10-W32-NEXT: v_cvt_i32_f32_e32 v2, v0 2881; GFX10-W32-NEXT: s_not_b32 exec_lo, exec_lo 2882; GFX10-W32-NEXT: v_mov_b32_e32 v2, 0 2883; GFX10-W32-NEXT: s_not_b32 exec_lo, exec_lo 2884; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 2885; GFX10-W32-NEXT: ds_swizzle_b32 v2, v2 offset:swizzle(SWAP,2) 2886; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 2887; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0) 2888; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2 2889; GFX10-W32-NEXT: v_cvt_f32_i32_e32 v1, v0 2890; GFX10-W32-NEXT: .LBB45_2: ; %ENDIF 2891; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s13 2892; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 2893; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 2894; GFX10-W32-NEXT: ; return to shader part epilog 2895main_body: 2896 %c.bc = bitcast i32 %c to float 2897 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 2898 %tex0 = extractelement <4 x float> %tex, i32 0 2899 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 2900 %cmp = icmp eq i32 %z, 0 2901 br i1 %cmp, label %IF, label %ENDIF 2902 2903IF: 2904 %dataf = extractelement <4 x float> %dtex, i32 0 2905 %data1 = fptosi float %dataf to i32 2906 %data2 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %data1, i32 0) 2907 %data3 = call i32 @llvm.amdgcn.ds.swizzle(i32 %data2, i32 2079) 2908 %data4 = call i32 @llvm.amdgcn.strict.wwm.i32(i32 %data3) 2909 %data4f = sitofp i32 %data4 to float 2910 br label %ENDIF 2911 2912ENDIF: 2913 %r = phi float [ %data4f, %IF ], [ 0.0, %main_body ] 2914 ret float %r 2915} 2916 2917; Check a case of a block being entirely WQM except for a bit of STRICT WQM. 2918define amdgpu_ps float @test_strict_wqm_within_wqm(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data) { 2919; GFX9-W64-LABEL: test_strict_wqm_within_wqm: 2920; GFX9-W64: ; %bb.0: ; %main_body 2921; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec 2922; GFX9-W64-NEXT: s_wqm_b64 exec, exec 2923; GFX9-W64-NEXT: s_mov_b64 s[14:15], exec 2924; GFX9-W64-NEXT: s_wqm_b64 exec, exec 2925; GFX9-W64-NEXT: v_mov_b32_e32 v2, v0 2926; GFX9-W64-NEXT: s_mov_b64 exec, s[14:15] 2927; GFX9-W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 2928; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 2929; GFX9-W64-NEXT: s_and_saveexec_b64 s[14:15], vcc 2930; GFX9-W64-NEXT: s_cbranch_execz .LBB46_2 2931; GFX9-W64-NEXT: ; %bb.1: ; %IF 2932; GFX9-W64-NEXT: s_mov_b64 s[16:17], exec 2933; GFX9-W64-NEXT: s_wqm_b64 exec, exec 2934; GFX9-W64-NEXT: image_sample v2, v2, s[0:7], s[8:11] dmask:0x1 2935; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2936; GFX9-W64-NEXT: image_sample v2, v2, s[0:7], s[8:11] dmask:0x1 2937; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2938; GFX9-W64-NEXT: v_cvt_i32_f32_e32 v2, v2 2939; GFX9-W64-NEXT: ds_swizzle_b32 v2, v2 offset:swizzle(SWAP,2) 2940; GFX9-W64-NEXT: s_mov_b64 exec, s[16:17] 2941; GFX9-W64-NEXT: s_waitcnt lgkmcnt(0) 2942; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2 2943; GFX9-W64-NEXT: v_cvt_f32_i32_e32 v0, v0 2944; GFX9-W64-NEXT: .LBB46_2: ; %ENDIF 2945; GFX9-W64-NEXT: s_or_b64 exec, exec, s[14:15] 2946; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 2947; GFX9-W64-NEXT: ; return to shader part epilog 2948; 2949; GFX10-W32-LABEL: test_strict_wqm_within_wqm: 2950; GFX10-W32: ; %bb.0: ; %main_body 2951; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo 2952; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 2953; GFX10-W32-NEXT: s_mov_b32 s13, exec_lo 2954; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 2955; GFX10-W32-NEXT: v_mov_b32_e32 v2, v0 2956; GFX10-W32-NEXT: s_mov_b32 exec_lo, s13 2957; GFX10-W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1 2958; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 2959; GFX10-W32-NEXT: s_and_saveexec_b32 s13, vcc_lo 2960; GFX10-W32-NEXT: s_cbranch_execz .LBB46_2 2961; GFX10-W32-NEXT: ; %bb.1: ; %IF 2962; GFX10-W32-NEXT: s_mov_b32 s14, exec_lo 2963; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 2964; GFX10-W32-NEXT: image_sample v2, v2, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 2965; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2966; GFX10-W32-NEXT: image_sample v2, v2, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 2967; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2968; GFX10-W32-NEXT: v_cvt_i32_f32_e32 v2, v2 2969; GFX10-W32-NEXT: ds_swizzle_b32 v2, v2 offset:swizzle(SWAP,2) 2970; GFX10-W32-NEXT: s_mov_b32 exec_lo, s14 2971; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0) 2972; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2 2973; GFX10-W32-NEXT: v_cvt_f32_i32_e32 v0, v0 2974; GFX10-W32-NEXT: .LBB46_2: ; %ENDIF 2975; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s13 2976; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 2977; GFX10-W32-NEXT: ; return to shader part epilog 2978main_body: 2979 %c.bc = bitcast i32 %c to float 2980 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 2981 %tex0 = extractelement <4 x float> %tex, i32 0 2982 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 2983 %cmp = icmp eq i32 %z, 0 2984 br i1 %cmp, label %IF, label %ENDIF 2985 2986IF: 2987 %dataf = extractelement <4 x float> %dtex, i32 0 2988 %data1 = fptosi float %dataf to i32 2989 %data2 = call i32 @llvm.amdgcn.ds.swizzle(i32 %data1, i32 2079) 2990 %data3 = call i32 @llvm.amdgcn.strict.wqm.i32(i32 %data2) 2991 %data3f = sitofp i32 %data3 to float 2992 br label %ENDIF 2993 2994ENDIF: 2995 %r = phi float [ %data3f, %IF ], [ 0.0, %main_body ] 2996 ret float %r 2997} 2998 2999;TODO: StrictWQM -> WQM transition could be improved. WQM could use the exec from the previous state instead of calling s_wqm again. 3000define amdgpu_ps float @test_strict_wqm_strict_wwm_wqm(i32 inreg %idx0, i32 inreg %idx1, <4 x i32> inreg %res, <4 x i32> inreg %res2, float %inp, <8 x i32> inreg %res3) { 3001; GFX9-W64-LABEL: test_strict_wqm_strict_wwm_wqm: 3002; GFX9-W64: ; %bb.0: ; %main_body 3003; GFX9-W64-NEXT: s_mov_b64 s[28:29], exec 3004; GFX9-W64-NEXT: s_mov_b32 s19, s17 3005; GFX9-W64-NEXT: s_mov_b64 s[30:31], exec 3006; GFX9-W64-NEXT: s_wqm_b64 exec, exec 3007; GFX9-W64-NEXT: s_mov_b32 s23, s5 3008; GFX9-W64-NEXT: s_mov_b32 s22, s4 3009; GFX9-W64-NEXT: s_mov_b32 s21, s3 3010; GFX9-W64-NEXT: s_mov_b32 s20, s2 3011; GFX9-W64-NEXT: s_mov_b32 s27, s9 3012; GFX9-W64-NEXT: s_mov_b32 s26, s8 3013; GFX9-W64-NEXT: s_mov_b32 s25, s7 3014; GFX9-W64-NEXT: s_mov_b32 s24, s6 3015; GFX9-W64-NEXT: s_mov_b32 s18, s16 3016; GFX9-W64-NEXT: s_mov_b32 s17, s15 3017; GFX9-W64-NEXT: s_mov_b32 s16, s14 3018; GFX9-W64-NEXT: s_mov_b32 s15, s13 3019; GFX9-W64-NEXT: s_mov_b32 s14, s12 3020; GFX9-W64-NEXT: s_mov_b32 s13, s11 3021; GFX9-W64-NEXT: s_mov_b32 s12, s10 3022; GFX9-W64-NEXT: v_mov_b32_e32 v1, s1 3023; GFX9-W64-NEXT: s_mov_b64 exec, s[30:31] 3024; GFX9-W64-NEXT: buffer_store_dword v0, v1, s[20:23], 0 idxen 3025; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec 3026; GFX9-W64-NEXT: s_wqm_b64 exec, exec 3027; GFX9-W64-NEXT: buffer_load_dword v2, v1, s[20:23], 0 idxen 3028; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 3029; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1 3030; GFX9-W64-NEXT: v_mov_b32_e32 v3, s0 3031; GFX9-W64-NEXT: buffer_load_dword v3, v3, s[24:27], 0 idxen 3032; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 3033; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec 3034; GFX9-W64-NEXT: s_wqm_b64 exec, exec 3035; GFX9-W64-NEXT: s_waitcnt vmcnt(1) 3036; GFX9-W64-NEXT: v_add_f32_e32 v2, v2, v2 3037; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1] 3038; GFX9-W64-NEXT: s_wqm_b64 exec, exec 3039; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2 3040; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v0 3041; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 3042; GFX9-W64-NEXT: v_mov_b32_e32 v4, v3 3043; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v4 3044; GFX9-W64-NEXT: s_and_b64 exec, exec, s[28:29] 3045; GFX9-W64-NEXT: image_sample v0, v0, s[12:19], s[20:23] dmask:0x1 3046; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 3047; GFX9-W64-NEXT: buffer_store_dword v0, v1, s[20:23], 0 idxen 3048; GFX9-W64-NEXT: buffer_load_dword v0, v1, s[20:23], 0 idxen 3049; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 3050; GFX9-W64-NEXT: ; return to shader part epilog 3051; 3052; GFX10-W32-LABEL: test_strict_wqm_strict_wwm_wqm: 3053; GFX10-W32: ; %bb.0: ; %main_body 3054; GFX10-W32-NEXT: s_mov_b32 s28, exec_lo 3055; GFX10-W32-NEXT: s_mov_b32 s19, s17 3056; GFX10-W32-NEXT: s_mov_b32 s29, exec_lo 3057; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 3058; GFX10-W32-NEXT: s_mov_b32 s23, s5 3059; GFX10-W32-NEXT: s_mov_b32 s22, s4 3060; GFX10-W32-NEXT: s_mov_b32 s21, s3 3061; GFX10-W32-NEXT: s_mov_b32 s20, s2 3062; GFX10-W32-NEXT: s_mov_b32 s27, s9 3063; GFX10-W32-NEXT: s_mov_b32 s26, s8 3064; GFX10-W32-NEXT: s_mov_b32 s25, s7 3065; GFX10-W32-NEXT: s_mov_b32 s24, s6 3066; GFX10-W32-NEXT: s_mov_b32 s18, s16 3067; GFX10-W32-NEXT: s_mov_b32 s17, s15 3068; GFX10-W32-NEXT: s_mov_b32 s16, s14 3069; GFX10-W32-NEXT: s_mov_b32 s15, s13 3070; GFX10-W32-NEXT: s_mov_b32 s14, s12 3071; GFX10-W32-NEXT: s_mov_b32 s13, s11 3072; GFX10-W32-NEXT: s_mov_b32 s12, s10 3073; GFX10-W32-NEXT: v_mov_b32_e32 v1, s1 3074; GFX10-W32-NEXT: s_mov_b32 exec_lo, s29 3075; GFX10-W32-NEXT: buffer_store_dword v0, v1, s[20:23], 0 idxen 3076; GFX10-W32-NEXT: s_mov_b32 s1, exec_lo 3077; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 3078; GFX10-W32-NEXT: buffer_load_dword v2, v1, s[20:23], 0 idxen 3079; GFX10-W32-NEXT: s_mov_b32 exec_lo, s1 3080; GFX10-W32-NEXT: s_or_saveexec_b32 s1, -1 3081; GFX10-W32-NEXT: v_mov_b32_e32 v3, s0 3082; GFX10-W32-NEXT: buffer_load_dword v3, v3, s[24:27], 0 idxen 3083; GFX10-W32-NEXT: s_mov_b32 exec_lo, s1 3084; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo 3085; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 3086; GFX10-W32-NEXT: s_waitcnt vmcnt(1) 3087; GFX10-W32-NEXT: v_add_f32_e32 v2, v2, v2 3088; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 3089; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 3090; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2 3091; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 3092; GFX10-W32-NEXT: v_mov_b32_e32 v4, v3 3093; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v0 3094; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v4 3095; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s28 3096; GFX10-W32-NEXT: image_sample v0, v0, s[12:19], s[20:23] dmask:0x1 dim:SQ_RSRC_IMG_1D 3097; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 3098; GFX10-W32-NEXT: buffer_store_dword v0, v1, s[20:23], 0 idxen 3099; GFX10-W32-NEXT: buffer_load_dword v0, v1, s[20:23], 0 idxen 3100; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 3101; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 3102; GFX10-W32-NEXT: ; return to shader part epilog 3103main_body: 3104 call void @llvm.amdgcn.struct.buffer.store.f32(float %inp, <4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0) 3105 %reload = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0) 3106 %temp = fadd float %reload, %reload 3107 %temp2 = call float @llvm.amdgcn.strict.wqm.f32(float %temp) 3108 %temp3 = fadd float %temp2, %temp2 3109 %reload_wwm = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res2, i32 %idx0, i32 0, i32 0, i32 0) 3110 %temp4 = call float @llvm.amdgcn.strict.wwm.f32(float %reload_wwm) 3111 %temp5 = fadd float %temp3, %temp4 3112 %tex = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %temp5, <8 x i32> %res3, <4 x i32> %res, i1 false, i32 0, i32 0) 3113 call void @llvm.amdgcn.struct.buffer.store.f32(float %tex, <4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0) 3114 %out = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0) 3115 ret float %out 3116} 3117 3118define amdgpu_ps float @test_strict_wwm_strict_wqm_wqm(i32 inreg %idx0, i32 inreg %idx1, <4 x i32> inreg %res, float %inp, <8 x i32> inreg %res2) { 3119; GFX9-W64-LABEL: test_strict_wwm_strict_wqm_wqm: 3120; GFX9-W64: ; %bb.0: ; %main_body 3121; GFX9-W64-NEXT: s_mov_b64 s[20:21], exec 3122; GFX9-W64-NEXT: s_mov_b32 s15, s13 3123; GFX9-W64-NEXT: s_mov_b64 s[22:23], exec 3124; GFX9-W64-NEXT: s_wqm_b64 exec, exec 3125; GFX9-W64-NEXT: s_mov_b32 s19, s5 3126; GFX9-W64-NEXT: s_mov_b32 s18, s4 3127; GFX9-W64-NEXT: s_mov_b32 s17, s3 3128; GFX9-W64-NEXT: s_mov_b32 s16, s2 3129; GFX9-W64-NEXT: s_mov_b32 s14, s12 3130; GFX9-W64-NEXT: s_mov_b32 s13, s11 3131; GFX9-W64-NEXT: s_mov_b32 s12, s10 3132; GFX9-W64-NEXT: s_mov_b32 s11, s9 3133; GFX9-W64-NEXT: s_mov_b32 s10, s8 3134; GFX9-W64-NEXT: s_mov_b32 s9, s7 3135; GFX9-W64-NEXT: s_mov_b32 s8, s6 3136; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 3137; GFX9-W64-NEXT: s_mov_b64 exec, s[22:23] 3138; GFX9-W64-NEXT: buffer_store_dword v0, v1, s[16:19], 0 idxen 3139; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1 3140; GFX9-W64-NEXT: v_mov_b32_e32 v2, s1 3141; GFX9-W64-NEXT: buffer_load_dword v2, v2, s[16:19], 0 idxen 3142; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 3143; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec 3144; GFX9-W64-NEXT: s_wqm_b64 exec, exec 3145; GFX9-W64-NEXT: buffer_load_dword v3, v1, s[16:19], 0 idxen 3146; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1] 3147; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], -1 3148; GFX9-W64-NEXT: s_waitcnt vmcnt(1) 3149; GFX9-W64-NEXT: v_add_f32_e32 v2, v2, v2 3150; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1] 3151; GFX9-W64-NEXT: s_wqm_b64 exec, exec 3152; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2 3153; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v0 3154; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 3155; GFX9-W64-NEXT: v_mov_b32_e32 v4, v3 3156; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v4 3157; GFX9-W64-NEXT: s_and_b64 exec, exec, s[20:21] 3158; GFX9-W64-NEXT: image_sample v0, v0, s[8:15], s[16:19] dmask:0x1 3159; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 3160; GFX9-W64-NEXT: buffer_store_dword v0, v1, s[16:19], 0 idxen 3161; GFX9-W64-NEXT: buffer_load_dword v0, v1, s[16:19], 0 idxen 3162; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 3163; GFX9-W64-NEXT: ; return to shader part epilog 3164; 3165; GFX10-W32-LABEL: test_strict_wwm_strict_wqm_wqm: 3166; GFX10-W32: ; %bb.0: ; %main_body 3167; GFX10-W32-NEXT: s_mov_b32 s20, exec_lo 3168; GFX10-W32-NEXT: s_mov_b32 s15, s13 3169; GFX10-W32-NEXT: s_mov_b32 s21, exec_lo 3170; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 3171; GFX10-W32-NEXT: s_mov_b32 s19, s5 3172; GFX10-W32-NEXT: s_mov_b32 s18, s4 3173; GFX10-W32-NEXT: s_mov_b32 s17, s3 3174; GFX10-W32-NEXT: s_mov_b32 s16, s2 3175; GFX10-W32-NEXT: s_mov_b32 s14, s12 3176; GFX10-W32-NEXT: s_mov_b32 s13, s11 3177; GFX10-W32-NEXT: s_mov_b32 s12, s10 3178; GFX10-W32-NEXT: s_mov_b32 s11, s9 3179; GFX10-W32-NEXT: s_mov_b32 s10, s8 3180; GFX10-W32-NEXT: s_mov_b32 s9, s7 3181; GFX10-W32-NEXT: s_mov_b32 s8, s6 3182; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 3183; GFX10-W32-NEXT: s_mov_b32 exec_lo, s21 3184; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 3185; GFX10-W32-NEXT: v_mov_b32_e32 v2, s1 3186; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 3187; GFX10-W32-NEXT: buffer_store_dword v0, v1, s[16:19], 0 idxen 3188; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 3189; GFX10-W32-NEXT: buffer_load_dword v2, v2, s[16:19], 0 idxen 3190; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 3191; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo 3192; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 3193; GFX10-W32-NEXT: buffer_load_dword v3, v1, s[16:19], 0 idxen 3194; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 3195; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 3196; GFX10-W32-NEXT: s_waitcnt vmcnt(1) 3197; GFX10-W32-NEXT: v_add_f32_e32 v2, v2, v2 3198; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 3199; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 3200; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2 3201; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 3202; GFX10-W32-NEXT: v_mov_b32_e32 v4, v3 3203; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v0 3204; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v4 3205; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s20 3206; GFX10-W32-NEXT: image_sample v0, v0, s[8:15], s[16:19] dmask:0x1 dim:SQ_RSRC_IMG_1D 3207; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 3208; GFX10-W32-NEXT: buffer_store_dword v0, v1, s[16:19], 0 idxen 3209; GFX10-W32-NEXT: buffer_load_dword v0, v1, s[16:19], 0 idxen 3210; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 3211; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 3212; GFX10-W32-NEXT: ; return to shader part epilog 3213main_body: 3214 call void @llvm.amdgcn.struct.buffer.store.f32(float %inp, <4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0) 3215 %reload = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0) 3216 %temp = fadd float %reload, %reload 3217 %temp2 = call float @llvm.amdgcn.strict.wwm.f32(float %temp) 3218 %temp3 = fadd float %temp2, %temp2 3219 %reload_wwm = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0) 3220 %temp4 = call float @llvm.amdgcn.strict.wqm.f32(float %reload_wwm) 3221 %temp5 = fadd float %temp3, %temp4 3222 %tex = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %temp5, <8 x i32> %res2, <4 x i32> %res, i1 false, i32 0, i32 0) 3223 call void @llvm.amdgcn.struct.buffer.store.f32(float %tex, <4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0) 3224 %out = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0) 3225 ret float %out 3226} 3227 3228;TODO: WQM -> StrictWQM transition could be improved. StrictWQM could use the exec from the previous state instead of calling s_wqm again. 3229define amdgpu_ps float @test_wqm_strict_wqm_wqm(i32 inreg %idx0, i32 inreg %idx1, <4 x i32> inreg %res, float %inp, <8 x i32> inreg %res2) { 3230; GFX9-W64-LABEL: test_wqm_strict_wqm_wqm: 3231; GFX9-W64: ; %bb.0: ; %main_body 3232; GFX9-W64-NEXT: s_mov_b64 s[20:21], exec 3233; GFX9-W64-NEXT: s_mov_b32 s15, s13 3234; GFX9-W64-NEXT: s_mov_b64 s[22:23], exec 3235; GFX9-W64-NEXT: s_wqm_b64 exec, exec 3236; GFX9-W64-NEXT: s_mov_b32 s19, s5 3237; GFX9-W64-NEXT: s_mov_b32 s18, s4 3238; GFX9-W64-NEXT: s_mov_b32 s17, s3 3239; GFX9-W64-NEXT: s_mov_b32 s16, s2 3240; GFX9-W64-NEXT: s_mov_b32 s14, s12 3241; GFX9-W64-NEXT: s_mov_b32 s13, s11 3242; GFX9-W64-NEXT: s_mov_b32 s12, s10 3243; GFX9-W64-NEXT: s_mov_b32 s11, s9 3244; GFX9-W64-NEXT: s_mov_b32 s10, s8 3245; GFX9-W64-NEXT: s_mov_b32 s9, s7 3246; GFX9-W64-NEXT: s_mov_b32 s8, s6 3247; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 3248; GFX9-W64-NEXT: s_mov_b64 exec, s[22:23] 3249; GFX9-W64-NEXT: buffer_store_dword v0, v1, s[16:19], 0 idxen 3250; GFX9-W64-NEXT: s_wqm_b64 exec, exec 3251; GFX9-W64-NEXT: v_mov_b32_e32 v0, s1 3252; GFX9-W64-NEXT: buffer_load_dword v0, v0, s[16:19], 0 idxen 3253; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec 3254; GFX9-W64-NEXT: s_wqm_b64 exec, exec 3255; GFX9-W64-NEXT: buffer_load_dword v2, v1, s[16:19], 0 idxen 3256; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1] 3257; GFX9-W64-NEXT: s_waitcnt vmcnt(1) 3258; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v0 3259; GFX9-W64-NEXT: image_sample v0, v0, s[8:15], s[16:19] dmask:0x1 3260; GFX9-W64-NEXT: s_waitcnt vmcnt(1) 3261; GFX9-W64-NEXT: v_mov_b32_e32 v3, v2 3262; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 3263; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v0 3264; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v3 3265; GFX9-W64-NEXT: s_and_b64 exec, exec, s[20:21] 3266; GFX9-W64-NEXT: image_sample v0, v0, s[8:15], s[16:19] dmask:0x1 3267; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 3268; GFX9-W64-NEXT: buffer_store_dword v0, v1, s[16:19], 0 idxen 3269; GFX9-W64-NEXT: buffer_load_dword v0, v1, s[16:19], 0 idxen 3270; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 3271; GFX9-W64-NEXT: ; return to shader part epilog 3272; 3273; GFX10-W32-LABEL: test_wqm_strict_wqm_wqm: 3274; GFX10-W32: ; %bb.0: ; %main_body 3275; GFX10-W32-NEXT: s_mov_b32 s20, exec_lo 3276; GFX10-W32-NEXT: s_mov_b32 s15, s13 3277; GFX10-W32-NEXT: s_mov_b32 s21, exec_lo 3278; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 3279; GFX10-W32-NEXT: s_mov_b32 s19, s5 3280; GFX10-W32-NEXT: s_mov_b32 s18, s4 3281; GFX10-W32-NEXT: s_mov_b32 s17, s3 3282; GFX10-W32-NEXT: s_mov_b32 s16, s2 3283; GFX10-W32-NEXT: s_mov_b32 s14, s12 3284; GFX10-W32-NEXT: s_mov_b32 s13, s11 3285; GFX10-W32-NEXT: s_mov_b32 s12, s10 3286; GFX10-W32-NEXT: s_mov_b32 s11, s9 3287; GFX10-W32-NEXT: s_mov_b32 s10, s8 3288; GFX10-W32-NEXT: s_mov_b32 s9, s7 3289; GFX10-W32-NEXT: s_mov_b32 s8, s6 3290; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 3291; GFX10-W32-NEXT: s_mov_b32 exec_lo, s21 3292; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 3293; GFX10-W32-NEXT: v_mov_b32_e32 v3, s1 3294; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s20 3295; GFX10-W32-NEXT: buffer_store_dword v0, v1, s[16:19], 0 idxen 3296; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 3297; GFX10-W32-NEXT: buffer_load_dword v0, v3, s[16:19], 0 idxen 3298; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo 3299; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 3300; GFX10-W32-NEXT: buffer_load_dword v2, v1, s[16:19], 0 idxen 3301; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 3302; GFX10-W32-NEXT: s_waitcnt vmcnt(1) 3303; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v0 3304; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 3305; GFX10-W32-NEXT: v_mov_b32_e32 v3, v2 3306; GFX10-W32-NEXT: image_sample v0, v0, s[8:15], s[16:19] dmask:0x1 dim:SQ_RSRC_IMG_1D 3307; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 3308; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v0 3309; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v3 3310; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s20 3311; GFX10-W32-NEXT: image_sample v0, v0, s[8:15], s[16:19] dmask:0x1 dim:SQ_RSRC_IMG_1D 3312; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 3313; GFX10-W32-NEXT: buffer_store_dword v0, v1, s[16:19], 0 idxen 3314; GFX10-W32-NEXT: buffer_load_dword v0, v1, s[16:19], 0 idxen 3315; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 3316; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 3317; GFX10-W32-NEXT: ; return to shader part epilog 3318main_body: 3319 call void @llvm.amdgcn.struct.buffer.store.f32(float %inp, <4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0) 3320 %reload = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0) 3321 %temp = fadd float %reload, %reload 3322 %tex = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %temp, <8 x i32> %res2, <4 x i32> %res, i1 false, i32 0, i32 0) 3323 %temp2 = fadd float %tex, %tex 3324 %reload_wwm = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0) 3325 %temp3 = call float @llvm.amdgcn.strict.wqm.f32(float %reload_wwm) 3326 %temp4 = fadd float %temp2, %temp3 3327 %tex2 = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %temp4, <8 x i32> %res2, <4 x i32> %res, i1 false, i32 0, i32 0) 3328 call void @llvm.amdgcn.struct.buffer.store.f32(float %tex2, <4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0) 3329 %out = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0) 3330 ret float %out 3331} 3332 3333declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #1 3334declare void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float>, i32, i32, <8 x i32>, i32, i32) #1 3335 3336declare void @llvm.amdgcn.struct.buffer.store.f32(float, <4 x i32>, i32, i32, i32, i32 immarg) #2 3337declare void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32, i32 immarg) #2 3338declare void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32 immarg) #2 3339declare void @llvm.amdgcn.raw.buffer.store.f32(float, <4 x i32>, i32, i32, i32 immarg) #2 3340declare float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32>, i32, i32, i32) #3 3341declare float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32>, i32, i32, i32, i32) #3 3342 3343declare <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32, i32, <8 x i32>, i32, i32) #3 3344declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #3 3345declare <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #3 3346declare float @llvm.amdgcn.image.sample.1d.f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #3 3347declare void @llvm.amdgcn.kill(i1) #1 3348declare float @llvm.amdgcn.wqm.f32(float) #3 3349declare i32 @llvm.amdgcn.wqm.i32(i32) #3 3350declare float @llvm.amdgcn.strict.wwm.f32(float) #3 3351declare i32 @llvm.amdgcn.strict.wwm.i32(i32) #3 3352declare float @llvm.amdgcn.wwm.f32(float) #3 3353declare i32 @llvm.amdgcn.wwm.i32(i32) #3 3354declare float @llvm.amdgcn.strict.wqm.f32(float) #3 3355declare i32 @llvm.amdgcn.strict.wqm.i32(i32) #3 3356declare i32 @llvm.amdgcn.set.inactive.i32(i32, i32) #4 3357declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #3 3358declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32) #3 3359declare <2 x half> @llvm.amdgcn.cvt.pkrtz(float, float) #3 3360declare void @llvm.amdgcn.exp.compr.v2f16(i32, i32, <2 x half>, <2 x half>, i1, i1) #1 3361declare float @llvm.amdgcn.interp.p1(float, i32, i32, i32) #2 3362declare float @llvm.amdgcn.interp.p2(float, float, i32, i32, i32) #2 3363declare i32 @llvm.amdgcn.ds.swizzle(i32, i32) 3364 3365attributes #1 = { nounwind } 3366attributes #2 = { nounwind readonly } 3367attributes #3 = { nounwind readnone } 3368attributes #4 = { nounwind readnone convergent } 3369attributes #5 = { "amdgpu-ps-wqm-outputs" } 3370attributes #6 = { nounwind "InitialPSInputAddr"="2" } 3371