1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py 2; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-W64 %s 3; RUN: llc -march=amdgcn -mcpu=gfx1030 -verify-machineinstrs -mattr=+wavefrontsize32,-wavefrontsize64 < %s | FileCheck -check-prefixes=GFX10-W32 %s 4 5; Check that WQM isn't triggered by image load/store intrinsics. 6define amdgpu_ps <4 x float> @test1(<8 x i32> inreg %rsrc, i32 %c) { 7; GFX9-W64-LABEL: test1: 8; GFX9-W64: ; %bb.0: ; %main_body 9; GFX9-W64-NEXT: v_mov_b32_e32 v4, v0 10; GFX9-W64-NEXT: image_load v[0:3], v0, s[0:7] dmask:0xf unorm 11; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 12; GFX9-W64-NEXT: image_store v[0:3], v4, s[0:7] dmask:0xf unorm 13; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 14; GFX9-W64-NEXT: ; return to shader part epilog 15; 16; GFX10-W32-LABEL: test1: 17; GFX10-W32: ; %bb.0: ; %main_body 18; GFX10-W32-NEXT: v_mov_b32_e32 v4, v0 19; GFX10-W32-NEXT: image_load v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm 20; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 21; GFX10-W32-NEXT: image_store v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm 22; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 23; GFX10-W32-NEXT: ; return to shader part epilog 24main_body: 25 %tex = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %c, <8 x i32> %rsrc, i32 0, i32 0) 26 call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> %tex, i32 15, i32 %c, <8 x i32> %rsrc, i32 0, i32 0) 27 ret <4 x float> %tex 28} 29 30; Check that WQM is triggered by code calculating inputs to image samples and is disabled as soon as possible 31define amdgpu_ps <4 x float> @test2(i32 inreg, i32 inreg, i32 inreg, i32 inreg %m0, <8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, <2 x float> %pos) #6 { 32; GFX9-W64-LABEL: test2: 33; GFX9-W64: ; %bb.0: ; %main_body 34; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec 35; GFX9-W64-NEXT: s_wqm_b64 exec, exec 36; GFX9-W64-NEXT: s_mov_b32 m0, s3 37; GFX9-W64-NEXT: s_nop 0 38; GFX9-W64-NEXT: v_interp_p1_f32_e32 v2, v0, attr0.x 39; GFX9-W64-NEXT: v_interp_p1_f32_e32 v3, v0, attr0.y 40; GFX9-W64-NEXT: v_interp_p2_f32_e32 v2, v1, attr0.x 41; GFX9-W64-NEXT: v_interp_p2_f32_e32 v3, v1, attr0.y 42; GFX9-W64-NEXT: s_and_b64 exec, exec, s[0:1] 43; GFX9-W64-NEXT: image_sample v[0:3], v[2:3], s[4:11], s[12:15] dmask:0xf 44; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 45; GFX9-W64-NEXT: ; return to shader part epilog 46; 47; GFX10-W32-LABEL: test2: 48; GFX10-W32: ; %bb.0: ; %main_body 49; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo 50; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 51; GFX10-W32-NEXT: s_mov_b32 m0, s3 52; GFX10-W32-NEXT: v_interp_p1_f32_e32 v2, v0, attr0.x 53; GFX10-W32-NEXT: v_interp_p1_f32_e32 v3, v0, attr0.y 54; GFX10-W32-NEXT: v_interp_p2_f32_e32 v2, v1, attr0.x 55; GFX10-W32-NEXT: v_interp_p2_f32_e32 v3, v1, attr0.y 56; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s0 57; GFX10-W32-NEXT: image_sample v[0:3], v[2:3], s[4:11], s[12:15] dmask:0xf dim:SQ_RSRC_IMG_2D 58; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 59; GFX10-W32-NEXT: ; return to shader part epilog 60main_body: 61 %inst23 = extractelement <2 x float> %pos, i32 0 62 %inst24 = extractelement <2 x float> %pos, i32 1 63 %inst25 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 0, i32 0, i32 %m0) 64 %inst26 = tail call float @llvm.amdgcn.interp.p2(float %inst25, float %inst24, i32 0, i32 0, i32 %m0) 65 %inst28 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 1, i32 0, i32 %m0) 66 %inst29 = tail call float @llvm.amdgcn.interp.p2(float %inst28, float %inst24, i32 1, i32 0, i32 %m0) 67 %tex = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32 15, float %inst26, float %inst29, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 68 ret <4 x float> %tex 69} 70 71; ... but disabled for stores (and, in this simple case, not re-enabled) ... 72define amdgpu_ps <4 x float> @test3(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float %c) { 73; GFX9-W64-LABEL: test3: 74; GFX9-W64: ; %bb.0: ; %main_body 75; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec 76; GFX9-W64-NEXT: s_wqm_b64 exec, exec 77; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 78; GFX9-W64-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf 79; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 80; GFX9-W64-NEXT: buffer_store_dwordx4 v[0:3], v0, s[0:3], 0 idxen 81; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 82; GFX9-W64-NEXT: ; return to shader part epilog 83; 84; GFX10-W32-LABEL: test3: 85; GFX10-W32: ; %bb.0: ; %main_body 86; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo 87; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 88; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 89; GFX10-W32-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D 90; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 91; GFX10-W32-NEXT: buffer_store_dwordx4 v[0:3], v0, s[0:3], 0 idxen 92; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 93; GFX10-W32-NEXT: ; return to shader part epilog 94main_body: 95 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 96 %tex.1 = bitcast <4 x float> %tex to <4 x i32> 97 %tex.2 = extractelement <4 x i32> %tex.1, i32 0 98 99 call void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float> %tex, <4 x i32> undef, i32 %tex.2, i32 0, i32 0, i32 0) 100 101 ret <4 x float> %tex 102} 103 104; ... and disabled for export. 105define amdgpu_ps void @test3x(i32 inreg, i32 inreg, i32 inreg, i32 inreg %m0, <8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, <2 x float> %pos) #6 { 106; GFX9-W64-LABEL: test3x: 107; GFX9-W64: ; %bb.0: ; %main_body 108; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec 109; GFX9-W64-NEXT: s_wqm_b64 exec, exec 110; GFX9-W64-NEXT: s_mov_b32 m0, s3 111; GFX9-W64-NEXT: s_nop 0 112; GFX9-W64-NEXT: v_interp_p1_f32_e32 v2, v0, attr0.x 113; GFX9-W64-NEXT: v_interp_p1_f32_e32 v3, v0, attr0.y 114; GFX9-W64-NEXT: v_interp_p2_f32_e32 v2, v1, attr0.x 115; GFX9-W64-NEXT: v_interp_p2_f32_e32 v3, v1, attr0.y 116; GFX9-W64-NEXT: s_and_b64 exec, exec, s[0:1] 117; GFX9-W64-NEXT: image_sample v[0:3], v[2:3], s[4:11], s[12:15] dmask:0xf 118; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 119; GFX9-W64-NEXT: exp mrt0 v0, v1, v2, v3 done vm 120; GFX9-W64-NEXT: s_endpgm 121; 122; GFX10-W32-LABEL: test3x: 123; GFX10-W32: ; %bb.0: ; %main_body 124; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo 125; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 126; GFX10-W32-NEXT: s_mov_b32 m0, s3 127; GFX10-W32-NEXT: v_interp_p1_f32_e32 v2, v0, attr0.x 128; GFX10-W32-NEXT: v_interp_p1_f32_e32 v3, v0, attr0.y 129; GFX10-W32-NEXT: v_interp_p2_f32_e32 v2, v1, attr0.x 130; GFX10-W32-NEXT: v_interp_p2_f32_e32 v3, v1, attr0.y 131; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s0 132; GFX10-W32-NEXT: image_sample v[0:3], v[2:3], s[4:11], s[12:15] dmask:0xf dim:SQ_RSRC_IMG_2D 133; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 134; GFX10-W32-NEXT: exp mrt0 v0, v1, v2, v3 done vm 135; GFX10-W32-NEXT: s_endpgm 136main_body: 137 %inst23 = extractelement <2 x float> %pos, i32 0 138 %inst24 = extractelement <2 x float> %pos, i32 1 139 %inst25 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 0, i32 0, i32 %m0) 140 %inst26 = tail call float @llvm.amdgcn.interp.p2(float %inst25, float %inst24, i32 0, i32 0, i32 %m0) 141 %inst28 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 1, i32 0, i32 %m0) 142 %inst29 = tail call float @llvm.amdgcn.interp.p2(float %inst28, float %inst24, i32 1, i32 0, i32 %m0) 143 %tex = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32 15, float %inst26, float %inst29, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 144 %tex.0 = extractelement <4 x float> %tex, i32 0 145 %tex.1 = extractelement <4 x float> %tex, i32 1 146 %tex.2 = extractelement <4 x float> %tex, i32 2 147 %tex.3 = extractelement <4 x float> %tex, i32 3 148 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %tex.0, float %tex.1, float %tex.2, float %tex.3, i1 true, i1 true) 149 ret void 150} 151 152; Check that WQM is re-enabled when required. 153define amdgpu_ps <4 x float> @test4(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float addrspace(1)* inreg %ptr, i32 %c, i32 %d, float %data) { 154; GFX9-W64-LABEL: test4: 155; GFX9-W64: ; %bb.0: ; %main_body 156; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec 157; GFX9-W64-NEXT: s_wqm_b64 exec, exec 158; GFX9-W64-NEXT: v_mul_lo_u32 v4, v0, v1 159; GFX9-W64-NEXT: image_sample v0, v4, s[0:7], s[8:11] dmask:0x1 160; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 161; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 162; GFX9-W64-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf 163; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 164; GFX9-W64-NEXT: buffer_store_dwordx4 v[0:3], v4, s[0:3], 0 idxen 165; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 166; GFX9-W64-NEXT: ; return to shader part epilog 167; 168; GFX10-W32-LABEL: test4: 169; GFX10-W32: ; %bb.0: ; %main_body 170; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo 171; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 172; GFX10-W32-NEXT: v_mul_lo_u32 v4, v0, v1 173; GFX10-W32-NEXT: image_sample v0, v4, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 174; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 175; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 176; GFX10-W32-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D 177; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 178; GFX10-W32-NEXT: buffer_store_dwordx4 v[0:3], v4, s[0:3], 0 idxen 179; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 180; GFX10-W32-NEXT: ; return to shader part epilog 181main_body: 182 %c.1 = mul i32 %c, %d 183 184 call void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float> undef, <4 x i32> undef, i32 %c.1, i32 0, i32 0, i32 0) 185 %c.1.bc = bitcast i32 %c.1 to float 186 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.1.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 187 %tex0 = extractelement <4 x float> %tex, i32 0 188 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 189 ret <4 x float> %dtex 190} 191 192; Check that WQM is triggered by the wqm intrinsic. 193; WQM was inserting an unecessary v_mov to self after the v_add. Make sure this 194; does not happen - the v_add should write the return reg directly. 195define amdgpu_ps float @test5(i32 inreg %idx0, i32 inreg %idx1) { 196; GFX9-W64-LABEL: test5: 197; GFX9-W64: ; %bb.0: ; %main_body 198; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec 199; GFX9-W64-NEXT: s_wqm_b64 exec, exec 200; GFX9-W64-NEXT: v_mov_b32_e32 v0, s0 201; GFX9-W64-NEXT: v_mov_b32_e32 v1, s1 202; GFX9-W64-NEXT: buffer_load_dword v0, v0, s[0:3], 0 idxen 203; GFX9-W64-NEXT: s_nop 0 204; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 205; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 206; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v1 207; GFX9-W64-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec killed $exec 208; GFX9-W64-NEXT: s_and_b64 exec, exec, s[2:3] 209; GFX9-W64-NEXT: ; return to shader part epilog 210; 211; GFX10-W32-LABEL: test5: 212; GFX10-W32: ; %bb.0: ; %main_body 213; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo 214; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 215; GFX10-W32-NEXT: v_mov_b32_e32 v0, s0 216; GFX10-W32-NEXT: v_mov_b32_e32 v1, s1 217; GFX10-W32-NEXT: s_clause 0x1 218; GFX10-W32-NEXT: buffer_load_dword v0, v0, s[0:3], 0 idxen 219; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 220; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 221; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v1 222; GFX10-W32-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec killed $exec 223; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s2 224; GFX10-W32-NEXT: ; return to shader part epilog 225main_body: 226 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 227 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 228 %out = fadd float %src0, %src1 229 %out.0 = call float @llvm.amdgcn.wqm.f32(float %out) 230 ret float %out.0 231} 232 233; Check that the wqm intrinsic works correctly for integers. 234define amdgpu_ps float @test6(i32 inreg %idx0, i32 inreg %idx1) { 235; GFX9-W64-LABEL: test6: 236; GFX9-W64: ; %bb.0: ; %main_body 237; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec 238; GFX9-W64-NEXT: s_wqm_b64 exec, exec 239; GFX9-W64-NEXT: v_mov_b32_e32 v0, s0 240; GFX9-W64-NEXT: v_mov_b32_e32 v1, s1 241; GFX9-W64-NEXT: buffer_load_dword v0, v0, s[0:3], 0 idxen 242; GFX9-W64-NEXT: s_nop 0 243; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 244; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 245; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v1 246; GFX9-W64-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec killed $exec 247; GFX9-W64-NEXT: s_and_b64 exec, exec, s[2:3] 248; GFX9-W64-NEXT: ; return to shader part epilog 249; 250; GFX10-W32-LABEL: test6: 251; GFX10-W32: ; %bb.0: ; %main_body 252; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo 253; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 254; GFX10-W32-NEXT: v_mov_b32_e32 v0, s0 255; GFX10-W32-NEXT: v_mov_b32_e32 v1, s1 256; GFX10-W32-NEXT: s_clause 0x1 257; GFX10-W32-NEXT: buffer_load_dword v0, v0, s[0:3], 0 idxen 258; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 259; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 260; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v1 261; GFX10-W32-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec killed $exec 262; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s2 263; GFX10-W32-NEXT: ; return to shader part epilog 264main_body: 265 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 266 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 267 %out = fadd float %src0, %src1 268 %out.0 = bitcast float %out to i32 269 %out.1 = call i32 @llvm.amdgcn.wqm.i32(i32 %out.0) 270 %out.2 = bitcast i32 %out.1 to float 271 ret float %out.2 272} 273 274; NOTE: llvm.amdgcn.wwm is deprecated, use llvm.amdgcn.strict.wwm instead. 275 276; Check that WWM is triggered by the wwm intrinsic. 277define amdgpu_ps float @test_wwm1(i32 inreg %idx0, i32 inreg %idx1) { 278; GFX9-W64-LABEL: test_wwm1: 279; GFX9-W64: ; %bb.0: ; %main_body 280; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1 281; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 282; GFX9-W64-NEXT: v_mov_b32_e32 v2, s1 283; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 284; GFX9-W64-NEXT: s_nop 0 285; GFX9-W64-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen 286; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 287; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v2 288; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 289; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 290; GFX9-W64-NEXT: ; return to shader part epilog 291; 292; GFX10-W32-LABEL: test_wwm1: 293; GFX10-W32: ; %bb.0: ; %main_body 294; GFX10-W32-NEXT: s_or_saveexec_b32 s2, -1 295; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 296; GFX10-W32-NEXT: v_mov_b32_e32 v2, s1 297; GFX10-W32-NEXT: s_clause 0x1 298; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 299; GFX10-W32-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen 300; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 301; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v2 302; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2 303; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 304; GFX10-W32-NEXT: ; return to shader part epilog 305main_body: 306 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 307 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 308 %out = fadd float %src0, %src1 309 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out) 310 ret float %out.0 311} 312 313; Same as above, but with an integer type. 314define amdgpu_ps float @test_wwm2(i32 inreg %idx0, i32 inreg %idx1) { 315; GFX9-W64-LABEL: test_wwm2: 316; GFX9-W64: ; %bb.0: ; %main_body 317; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1 318; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 319; GFX9-W64-NEXT: v_mov_b32_e32 v2, s1 320; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 321; GFX9-W64-NEXT: s_nop 0 322; GFX9-W64-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen 323; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 324; GFX9-W64-NEXT: v_add_u32_e32 v1, v1, v2 325; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 326; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 327; GFX9-W64-NEXT: ; return to shader part epilog 328; 329; GFX10-W32-LABEL: test_wwm2: 330; GFX10-W32: ; %bb.0: ; %main_body 331; GFX10-W32-NEXT: s_or_saveexec_b32 s2, -1 332; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 333; GFX10-W32-NEXT: v_mov_b32_e32 v2, s1 334; GFX10-W32-NEXT: s_clause 0x1 335; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 336; GFX10-W32-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen 337; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 338; GFX10-W32-NEXT: v_add_nc_u32_e32 v1, v1, v2 339; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2 340; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 341; GFX10-W32-NEXT: ; return to shader part epilog 342main_body: 343 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 344 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 345 %src0.0 = bitcast float %src0 to i32 346 %src1.0 = bitcast float %src1 to i32 347 %out = add i32 %src0.0, %src1.0 348 %out.0 = call i32 @llvm.amdgcn.wwm.i32(i32 %out) 349 %out.1 = bitcast i32 %out.0 to float 350 ret float %out.1 351} 352 353; Check that we don't leave WWM on for computations that don't require WWM, 354; since that will lead clobbering things that aren't supposed to be clobbered 355; in cases like this. 356; We enforce this by checking that v_add gets emitted in the same block as 357; WWM computations. 358define amdgpu_ps float @test_wwm3(i32 inreg %idx) { 359; GFX9-W64-LABEL: test_wwm3: 360; GFX9-W64: ; %bb.0: ; %main_body 361; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 362; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 363; GFX9-W64-NEXT: v_cmp_gt_u32_e32 vcc, 16, v0 364; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 365; GFX9-W64-NEXT: s_and_saveexec_b64 s[2:3], vcc 366; GFX9-W64-NEXT: s_cbranch_execz .LBB9_2 367; GFX9-W64-NEXT: ; %bb.1: ; %if 368; GFX9-W64-NEXT: s_or_saveexec_b64 s[4:5], -1 369; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 370; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 371; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 372; GFX9-W64-NEXT: v_add_f32_e32 v2, v1, v1 373; GFX9-W64-NEXT: s_mov_b64 exec, s[4:5] 374; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2 375; GFX9-W64-NEXT: v_add_f32_e32 v0, v1, v0 376; GFX9-W64-NEXT: .LBB9_2: ; %endif 377; GFX9-W64-NEXT: s_or_b64 exec, exec, s[2:3] 378; GFX9-W64-NEXT: ; return to shader part epilog 379; 380; GFX10-W32-LABEL: test_wwm3: 381; GFX10-W32: ; %bb.0: ; %main_body 382; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 383; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 384; GFX10-W32-NEXT: v_cmp_gt_u32_e32 vcc_lo, 16, v0 385; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 386; GFX10-W32-NEXT: s_and_saveexec_b32 s1, vcc_lo 387; GFX10-W32-NEXT: s_cbranch_execz .LBB9_2 388; GFX10-W32-NEXT: ; %bb.1: ; %if 389; GFX10-W32-NEXT: s_or_saveexec_b32 s2, -1 390; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 391; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 392; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 393; GFX10-W32-NEXT: v_add_f32_e32 v2, v1, v1 394; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2 395; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2 396; GFX10-W32-NEXT: v_add_f32_e32 v0, v1, v0 397; GFX10-W32-NEXT: .LBB9_2: ; %endif 398; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s1 399; GFX10-W32-NEXT: ; return to shader part epilog 400main_body: 401 ; use mbcnt to make sure the branch is divergent 402 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 403 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 404 %cc = icmp uge i32 %hi, 16 405 br i1 %cc, label %endif, label %if 406 407if: 408 %src = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 409 %out = fadd float %src, %src 410 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out) 411 %out.1 = fadd float %src, %out.0 412 br label %endif 413 414endif: 415 %out.2 = phi float [ %out.1, %if ], [ 0.0, %main_body ] 416 ret float %out.2 417} 418 419; Check that WWM writes aren't coalesced with non-WWM writes, since the WWM 420; write could clobber disabled channels in the non-WWM one. 421; We enforce this by checking that v_mov gets emitted in the same block as 422; WWM computations. 423define amdgpu_ps float @test_wwm4(i32 inreg %idx) { 424; GFX9-W64-LABEL: test_wwm4: 425; GFX9-W64: ; %bb.0: ; %main_body 426; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 427; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 428; GFX9-W64-NEXT: v_cmp_gt_u32_e32 vcc, 16, v0 429; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 430; GFX9-W64-NEXT: s_and_saveexec_b64 s[2:3], vcc 431; GFX9-W64-NEXT: s_cbranch_execz .LBB10_2 432; GFX9-W64-NEXT: ; %bb.1: ; %if 433; GFX9-W64-NEXT: s_or_saveexec_b64 s[4:5], -1 434; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 435; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 436; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 437; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v1 438; GFX9-W64-NEXT: s_mov_b64 exec, s[4:5] 439; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 440; GFX9-W64-NEXT: .LBB10_2: ; %endif 441; GFX9-W64-NEXT: s_or_b64 exec, exec, s[2:3] 442; GFX9-W64-NEXT: ; return to shader part epilog 443; 444; GFX10-W32-LABEL: test_wwm4: 445; GFX10-W32: ; %bb.0: ; %main_body 446; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 447; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 448; GFX10-W32-NEXT: v_cmp_gt_u32_e32 vcc_lo, 16, v0 449; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 450; GFX10-W32-NEXT: s_and_saveexec_b32 s1, vcc_lo 451; GFX10-W32-NEXT: s_cbranch_execz .LBB10_2 452; GFX10-W32-NEXT: ; %bb.1: ; %if 453; GFX10-W32-NEXT: s_or_saveexec_b32 s2, -1 454; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 455; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 456; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 457; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v1 458; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2 459; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 460; GFX10-W32-NEXT: .LBB10_2: ; %endif 461; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s1 462; GFX10-W32-NEXT: ; return to shader part epilog 463main_body: 464 ; use mbcnt to make sure the branch is divergent 465 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 466 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 467 %cc = icmp uge i32 %hi, 16 468 br i1 %cc, label %endif, label %if 469 470if: 471 %src = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 472 %out = fadd float %src, %src 473 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out) 474 br label %endif 475 476endif: 477 %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ] 478 ret float %out.1 479} 480 481; Make sure the transition from Exact to WWM then WQM works properly. 482define amdgpu_ps float @test_wwm5(i32 inreg %idx0, i32 inreg %idx1) { 483; GFX9-W64-LABEL: test_wwm5: 484; GFX9-W64: ; %bb.0: ; %main_body 485; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec 486; GFX9-W64-NEXT: v_mov_b32_e32 v0, s0 487; GFX9-W64-NEXT: buffer_load_dword v2, v0, s[0:3], 0 idxen 488; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 489; GFX9-W64-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen 490; GFX9-W64-NEXT: s_or_saveexec_b64 s[4:5], -1 491; GFX9-W64-NEXT: v_mov_b32_e32 v1, s1 492; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 493; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 494; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v1 495; GFX9-W64-NEXT: s_mov_b64 exec, s[4:5] 496; GFX9-W64-NEXT: s_wqm_b64 exec, exec 497; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 498; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v0 499; GFX9-W64-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec killed $exec 500; GFX9-W64-NEXT: s_and_b64 exec, exec, s[2:3] 501; GFX9-W64-NEXT: ; return to shader part epilog 502; 503; GFX10-W32-LABEL: test_wwm5: 504; GFX10-W32: ; %bb.0: ; %main_body 505; GFX10-W32-NEXT: v_mov_b32_e32 v0, s0 506; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo 507; GFX10-W32-NEXT: buffer_load_dword v2, v0, s[0:3], 0 idxen 508; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 509; GFX10-W32-NEXT: v_mov_b32_e32 v1, s1 510; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 511; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 512; GFX10-W32-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen 513; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 514; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 515; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 516; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v1 517; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 518; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 519; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 520; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v0 521; GFX10-W32-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec killed $exec 522; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s2 523; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 524; GFX10-W32-NEXT: ; return to shader part epilog 525main_body: 526 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 527 call void @llvm.amdgcn.struct.buffer.store.f32(float %src0, <4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 528 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 529 %temp = fadd float %src1, %src1 530 %temp.0 = call float @llvm.amdgcn.wwm.f32(float %temp) 531 %out = fadd float %temp.0, %temp.0 532 %out.0 = call float @llvm.amdgcn.wqm.f32(float %out) 533 ret float %out.0 534} 535 536; Check that WWM is turned on correctly across basic block boundaries. 537; if..then..endif version 538;SI-CHECK: buffer_load_dword 539;VI-CHECK: flat_load_dword 540;SI-CHECK: buffer_load_dword 541;VI-CHECK: flat_load_dword 542define amdgpu_ps float @test_wwm6_then() { 543; GFX9-W64-LABEL: test_wwm6_then: 544; GFX9-W64: ; %bb.0: ; %main_body 545; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], -1 546; GFX9-W64-NEXT: global_load_dword v1, v[3:4], off glc 547; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 548; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1] 549; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 550; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 551; GFX9-W64-NEXT: v_cmp_gt_u32_e32 vcc, 16, v0 552; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 553; GFX9-W64-NEXT: s_and_saveexec_b64 s[0:1], vcc 554; GFX9-W64-NEXT: s_cbranch_execz .LBB12_2 555; GFX9-W64-NEXT: ; %bb.1: ; %if 556; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1 557; GFX9-W64-NEXT: global_load_dword v2, v[3:4], off glc 558; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 559; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v2 560; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 561; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 562; GFX9-W64-NEXT: .LBB12_2: ; %endif 563; GFX9-W64-NEXT: s_or_b64 exec, exec, s[0:1] 564; GFX9-W64-NEXT: ; return to shader part epilog 565; 566; GFX10-W32-LABEL: test_wwm6_then: 567; GFX10-W32: ; %bb.0: ; %main_body 568; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 569; GFX10-W32-NEXT: global_load_dword v1, v[3:4], off glc dlc 570; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 571; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 572; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 573; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 574; GFX10-W32-NEXT: v_cmp_gt_u32_e32 vcc_lo, 16, v0 575; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 576; GFX10-W32-NEXT: s_and_saveexec_b32 s0, vcc_lo 577; GFX10-W32-NEXT: s_cbranch_execz .LBB12_2 578; GFX10-W32-NEXT: ; %bb.1: ; %if 579; GFX10-W32-NEXT: s_or_saveexec_b32 s1, -1 580; GFX10-W32-NEXT: global_load_dword v2, v[3:4], off glc dlc 581; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 582; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v2 583; GFX10-W32-NEXT: s_mov_b32 exec_lo, s1 584; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 585; GFX10-W32-NEXT: .LBB12_2: ; %endif 586; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s0 587; GFX10-W32-NEXT: ; return to shader part epilog 588main_body: 589 %src0 = load volatile float, float addrspace(1)* undef 590 ; use mbcnt to make sure the branch is divergent 591 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 592 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 593 %cc = icmp uge i32 %hi, 16 594 br i1 %cc, label %endif, label %if 595 596if: 597 %src1 = load volatile float, float addrspace(1)* undef 598 %out = fadd float %src0, %src1 599 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out) 600 br label %endif 601 602endif: 603 %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ] 604 ret float %out.1 605} 606 607; Check that WWM is turned on correctly across basic block boundaries. 608; loop version 609;SI-CHECK: buffer_load_dword 610;VI-CHECK: flat_load_dword 611;SI-CHECK: buffer_load_dword 612;VI-CHECK: flat_load_dword 613define amdgpu_ps float @test_wwm6_loop() { 614; GFX9-W64-LABEL: test_wwm6_loop: 615; GFX9-W64: ; %bb.0: ; %main_body 616; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], -1 617; GFX9-W64-NEXT: global_load_dword v1, v[3:4], off glc 618; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 619; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1] 620; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 621; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v0 622; GFX9-W64-NEXT: s_mov_b64 s[0:1], 0 623; GFX9-W64-NEXT: .LBB13_1: ; %loop 624; GFX9-W64-NEXT: ; =>This Inner Loop Header: Depth=1 625; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1 626; GFX9-W64-NEXT: global_load_dword v2, v[3:4], off glc 627; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 628; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 629; GFX9-W64-NEXT: v_add_u32_e32 v3, -1, v3 630; GFX9-W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3 631; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1 632; GFX9-W64-NEXT: v_add_f32_e32 v2, v1, v2 633; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 634; GFX9-W64-NEXT: s_or_b64 s[0:1], vcc, s[0:1] 635; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2 636; GFX9-W64-NEXT: s_andn2_b64 exec, exec, s[0:1] 637; GFX9-W64-NEXT: s_cbranch_execnz .LBB13_1 638; GFX9-W64-NEXT: ; %bb.2: ; %endloop 639; GFX9-W64-NEXT: s_or_b64 exec, exec, s[0:1] 640; GFX9-W64-NEXT: ; return to shader part epilog 641; 642; GFX10-W32-LABEL: test_wwm6_loop: 643; GFX10-W32: ; %bb.0: ; %main_body 644; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 645; GFX10-W32-NEXT: global_load_dword v1, v[3:4], off glc dlc 646; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 647; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 648; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 649; GFX10-W32-NEXT: s_mov_b32 s0, 0 650; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v0 651; GFX10-W32-NEXT: .LBB13_1: ; %loop 652; GFX10-W32-NEXT: ; =>This Inner Loop Header: Depth=1 653; GFX10-W32-NEXT: s_or_saveexec_b32 s1, -1 654; GFX10-W32-NEXT: global_load_dword v2, v[3:4], off glc dlc 655; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 656; GFX10-W32-NEXT: s_mov_b32 exec_lo, s1 657; GFX10-W32-NEXT: v_add_nc_u32_e32 v3, -1, v3 658; GFX10-W32-NEXT: s_or_saveexec_b32 s1, -1 659; GFX10-W32-NEXT: v_add_f32_e32 v2, v1, v2 660; GFX10-W32-NEXT: s_mov_b32 exec_lo, s1 661; GFX10-W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v3 662; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2 663; GFX10-W32-NEXT: s_or_b32 s0, vcc_lo, s0 664; GFX10-W32-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 665; GFX10-W32-NEXT: s_cbranch_execnz .LBB13_1 666; GFX10-W32-NEXT: ; %bb.2: ; %endloop 667; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s0 668; GFX10-W32-NEXT: ; return to shader part epilog 669main_body: 670 %src0 = load volatile float, float addrspace(1)* undef 671 ; use mbcnt to make sure the branch is divergent 672 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 673 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 674 br label %loop 675 676loop: 677 %counter = phi i32 [ %hi, %main_body ], [ %counter.1, %loop ] 678 %src1 = load volatile float, float addrspace(1)* undef 679 %out = fadd float %src0, %src1 680 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out) 681 %counter.1 = sub i32 %counter, 1 682 %cc = icmp ne i32 %counter.1, 0 683 br i1 %cc, label %loop, label %endloop 684 685endloop: 686 ret float %out.0 687} 688 689; Check that @llvm.amdgcn.set.inactive disables WWM. 690define amdgpu_ps void @test_wwm_set_inactive1(i32 inreg %idx) { 691; GFX9-W64-LABEL: test_wwm_set_inactive1: 692; GFX9-W64: ; %bb.0: ; %main_body 693; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 694; GFX9-W64-NEXT: buffer_load_dword v2, v1, s[0:3], 0 idxen 695; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 696; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2 697; GFX9-W64-NEXT: s_not_b64 exec, exec 698; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 699; GFX9-W64-NEXT: s_not_b64 exec, exec 700; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], -1 701; GFX9-W64-NEXT: v_add_u32_e32 v0, v0, v0 702; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1] 703; GFX9-W64-NEXT: v_mov_b32_e32 v2, v0 704; GFX9-W64-NEXT: buffer_store_dword v2, v1, s[0:3], 0 idxen 705; GFX9-W64-NEXT: s_endpgm 706; 707; GFX10-W32-LABEL: test_wwm_set_inactive1: 708; GFX10-W32: ; %bb.0: ; %main_body 709; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 710; GFX10-W32-NEXT: buffer_load_dword v2, v1, s[0:3], 0 idxen 711; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 712; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2 713; GFX10-W32-NEXT: s_not_b32 exec_lo, exec_lo 714; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 715; GFX10-W32-NEXT: s_not_b32 exec_lo, exec_lo 716; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 717; GFX10-W32-NEXT: v_add_nc_u32_e32 v0, v0, v0 718; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 719; GFX10-W32-NEXT: v_mov_b32_e32 v2, v0 720; GFX10-W32-NEXT: buffer_store_dword v2, v1, s[0:3], 0 idxen 721; GFX10-W32-NEXT: s_endpgm 722main_body: 723 %src = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 724 %src.0 = bitcast float %src to i32 725 %src.1 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %src.0, i32 0) 726 %out = add i32 %src.1, %src.1 727 %out.0 = call i32 @llvm.amdgcn.wwm.i32(i32 %out) 728 %out.1 = bitcast i32 %out.0 to float 729 call void @llvm.amdgcn.struct.buffer.store.f32(float %out.1, <4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 730 ret void 731} 732 733; Check that Strict WQM is triggered by the strict_wqm intrinsic. 734define amdgpu_ps float @test_strict_wqm1(i32 inreg %idx0, i32 inreg %idx1) { 735; GFX9-W64-LABEL: test_strict_wqm1: 736; GFX9-W64: ; %bb.0: ; %main_body 737; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec 738; GFX9-W64-NEXT: s_wqm_b64 exec, exec 739; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 740; GFX9-W64-NEXT: v_mov_b32_e32 v2, s1 741; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 742; GFX9-W64-NEXT: s_nop 0 743; GFX9-W64-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen 744; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 745; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v2 746; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 747; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 748; GFX9-W64-NEXT: ; return to shader part epilog 749; 750; GFX10-W32-LABEL: test_strict_wqm1: 751; GFX10-W32: ; %bb.0: ; %main_body 752; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo 753; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 754; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 755; GFX10-W32-NEXT: v_mov_b32_e32 v2, s1 756; GFX10-W32-NEXT: s_clause 0x1 757; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 758; GFX10-W32-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen 759; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 760; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v2 761; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2 762; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 763; GFX10-W32-NEXT: ; return to shader part epilog 764main_body: 765 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 766 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 767 %out = fadd float %src0, %src1 768 %out.0 = call float @llvm.amdgcn.strict.wqm.f32(float %out) 769 ret float %out.0 770} 771 772; Same as above, but with an integer type. 773define amdgpu_ps float @test_strict_wqm2(i32 inreg %idx0, i32 inreg %idx1) { 774; GFX9-W64-LABEL: test_strict_wqm2: 775; GFX9-W64: ; %bb.0: ; %main_body 776; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec 777; GFX9-W64-NEXT: s_wqm_b64 exec, exec 778; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 779; GFX9-W64-NEXT: v_mov_b32_e32 v2, s1 780; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 781; GFX9-W64-NEXT: s_nop 0 782; GFX9-W64-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen 783; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 784; GFX9-W64-NEXT: v_add_u32_e32 v1, v1, v2 785; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 786; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 787; GFX9-W64-NEXT: ; return to shader part epilog 788; 789; GFX10-W32-LABEL: test_strict_wqm2: 790; GFX10-W32: ; %bb.0: ; %main_body 791; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo 792; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 793; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 794; GFX10-W32-NEXT: v_mov_b32_e32 v2, s1 795; GFX10-W32-NEXT: s_clause 0x1 796; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 797; GFX10-W32-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen 798; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 799; GFX10-W32-NEXT: v_add_nc_u32_e32 v1, v1, v2 800; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2 801; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 802; GFX10-W32-NEXT: ; return to shader part epilog 803main_body: 804 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 805 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 806 %src0.0 = bitcast float %src0 to i32 807 %src1.0 = bitcast float %src1 to i32 808 %out = add i32 %src0.0, %src1.0 809 %out.0 = call i32 @llvm.amdgcn.strict.wqm.i32(i32 %out) 810 %out.1 = bitcast i32 %out.0 to float 811 ret float %out.1 812} 813 814; Check that we don't leave Strict WQM on for computations that don't require it, 815; since that will lead clobbering things that aren't supposed to be clobbered 816; in cases like this. 817; We enforce this by checking that v_add gets emitted in the same block as 818; WWM computations. 819define amdgpu_ps float @test_strict_wqm3(i32 inreg %idx) { 820; GFX9-W64-LABEL: test_strict_wqm3: 821; GFX9-W64: ; %bb.0: ; %main_body 822; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 823; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 824; GFX9-W64-NEXT: v_cmp_gt_u32_e32 vcc, 16, v0 825; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 826; GFX9-W64-NEXT: s_and_saveexec_b64 s[2:3], vcc 827; GFX9-W64-NEXT: s_cbranch_execz .LBB17_2 828; GFX9-W64-NEXT: ; %bb.1: ; %if 829; GFX9-W64-NEXT: s_mov_b64 s[4:5], exec 830; GFX9-W64-NEXT: s_wqm_b64 exec, exec 831; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 832; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 833; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 834; GFX9-W64-NEXT: v_add_f32_e32 v2, v1, v1 835; GFX9-W64-NEXT: s_mov_b64 exec, s[4:5] 836; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2 837; GFX9-W64-NEXT: v_add_f32_e32 v0, v1, v0 838; GFX9-W64-NEXT: .LBB17_2: ; %endif 839; GFX9-W64-NEXT: s_or_b64 exec, exec, s[2:3] 840; GFX9-W64-NEXT: ; return to shader part epilog 841; 842; GFX10-W32-LABEL: test_strict_wqm3: 843; GFX10-W32: ; %bb.0: ; %main_body 844; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 845; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 846; GFX10-W32-NEXT: v_cmp_gt_u32_e32 vcc_lo, 16, v0 847; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 848; GFX10-W32-NEXT: s_and_saveexec_b32 s1, vcc_lo 849; GFX10-W32-NEXT: s_cbranch_execz .LBB17_2 850; GFX10-W32-NEXT: ; %bb.1: ; %if 851; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo 852; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 853; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 854; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 855; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 856; GFX10-W32-NEXT: v_add_f32_e32 v2, v1, v1 857; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2 858; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2 859; GFX10-W32-NEXT: v_add_f32_e32 v0, v1, v0 860; GFX10-W32-NEXT: .LBB17_2: ; %endif 861; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s1 862; GFX10-W32-NEXT: ; return to shader part epilog 863main_body: 864 ; use mbcnt to make sure the branch is divergent 865 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 866 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 867 %cc = icmp uge i32 %hi, 16 868 br i1 %cc, label %endif, label %if 869 870if: 871 %src = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 872 %out = fadd float %src, %src 873 %out.0 = call float @llvm.amdgcn.strict.wqm.f32(float %out) 874 %out.1 = fadd float %src, %out.0 875 br label %endif 876 877endif: 878 %out.2 = phi float [ %out.1, %if ], [ 0.0, %main_body ] 879 ret float %out.2 880} 881 882; Check that Strict WQM writes aren't coalesced with non-strict writes, since 883; the Strict WQM write could clobber disabled channels in the non-strict one. 884; We enforce this by checking that v_mov gets emitted in the same block as 885; WWM computations. 886define amdgpu_ps float @test_strict_wqm4(i32 inreg %idx) { 887; GFX9-W64-LABEL: test_strict_wqm4: 888; GFX9-W64: ; %bb.0: ; %main_body 889; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 890; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 891; GFX9-W64-NEXT: v_cmp_gt_u32_e32 vcc, 16, v0 892; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 893; GFX9-W64-NEXT: s_and_saveexec_b64 s[2:3], vcc 894; GFX9-W64-NEXT: s_cbranch_execz .LBB18_2 895; GFX9-W64-NEXT: ; %bb.1: ; %if 896; GFX9-W64-NEXT: s_mov_b64 s[4:5], exec 897; GFX9-W64-NEXT: s_wqm_b64 exec, exec 898; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 899; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 900; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 901; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v1 902; GFX9-W64-NEXT: s_mov_b64 exec, s[4:5] 903; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 904; GFX9-W64-NEXT: .LBB18_2: ; %endif 905; GFX9-W64-NEXT: s_or_b64 exec, exec, s[2:3] 906; GFX9-W64-NEXT: ; return to shader part epilog 907; 908; GFX10-W32-LABEL: test_strict_wqm4: 909; GFX10-W32: ; %bb.0: ; %main_body 910; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 911; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 912; GFX10-W32-NEXT: v_cmp_gt_u32_e32 vcc_lo, 16, v0 913; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 914; GFX10-W32-NEXT: s_and_saveexec_b32 s1, vcc_lo 915; GFX10-W32-NEXT: s_cbranch_execz .LBB18_2 916; GFX10-W32-NEXT: ; %bb.1: ; %if 917; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo 918; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 919; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 920; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 921; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 922; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v1 923; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2 924; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 925; GFX10-W32-NEXT: .LBB18_2: ; %endif 926; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s1 927; GFX10-W32-NEXT: ; return to shader part epilog 928main_body: 929 ; use mbcnt to make sure the branch is divergent 930 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 931 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 932 %cc = icmp uge i32 %hi, 16 933 br i1 %cc, label %endif, label %if 934 935if: 936 %src = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 937 %out = fadd float %src, %src 938 %out.0 = call float @llvm.amdgcn.strict.wqm.f32(float %out) 939 br label %endif 940 941endif: 942 %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ] 943 ret float %out.1 944} 945 946; Make sure the transition from Exact to Strict WQM then WQM works properly. 947define amdgpu_ps float @test_strict_wqm5(i32 inreg %idx0, i32 inreg %idx1) { 948; GFX9-W64-LABEL: test_strict_wqm5: 949; GFX9-W64: ; %bb.0: ; %main_body 950; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec 951; GFX9-W64-NEXT: v_mov_b32_e32 v0, s0 952; GFX9-W64-NEXT: buffer_load_dword v2, v0, s[0:3], 0 idxen 953; GFX9-W64-NEXT: s_mov_b64 s[4:5], exec 954; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 955; GFX9-W64-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen 956; GFX9-W64-NEXT: s_wqm_b64 exec, exec 957; GFX9-W64-NEXT: v_mov_b32_e32 v1, s1 958; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 959; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 960; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v1 961; GFX9-W64-NEXT: s_mov_b64 exec, s[4:5] 962; GFX9-W64-NEXT: s_wqm_b64 exec, exec 963; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 964; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v0 965; GFX9-W64-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec killed $exec 966; GFX9-W64-NEXT: s_and_b64 exec, exec, s[2:3] 967; GFX9-W64-NEXT: ; return to shader part epilog 968; 969; GFX10-W32-LABEL: test_strict_wqm5: 970; GFX10-W32: ; %bb.0: ; %main_body 971; GFX10-W32-NEXT: v_mov_b32_e32 v0, s0 972; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo 973; GFX10-W32-NEXT: buffer_load_dword v2, v0, s[0:3], 0 idxen 974; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo 975; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 976; GFX10-W32-NEXT: v_mov_b32_e32 v1, s1 977; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 978; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 979; GFX10-W32-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen 980; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo 981; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 982; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 983; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 984; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v1 985; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 986; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 987; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 988; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v0 989; GFX10-W32-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec killed $exec 990; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s2 991; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 992; GFX10-W32-NEXT: ; return to shader part epilog 993main_body: 994 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 995 call void @llvm.amdgcn.struct.buffer.store.f32(float %src0, <4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 996 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 997 %temp = fadd float %src1, %src1 998 %temp.0 = call float @llvm.amdgcn.strict.wqm.f32(float %temp) 999 %out = fadd float %temp.0, %temp.0 1000 %out.0 = call float @llvm.amdgcn.wqm.f32(float %out) 1001 ret float %out.0 1002} 1003 1004; Check that Strict WQM is turned on correctly across basic block boundaries. 1005; if..then..endif version 1006;SI-CHECK: buffer_load_dword 1007;VI-CHECK: flat_load_dword 1008;SI-CHECK: buffer_load_dword 1009;VI-CHECK: flat_load_dword 1010define amdgpu_ps float @test_strict_wqm6_then() { 1011; GFX9-W64-LABEL: test_strict_wqm6_then: 1012; GFX9-W64: ; %bb.0: ; %main_body 1013; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec 1014; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1015; GFX9-W64-NEXT: global_load_dword v1, v[3:4], off glc 1016; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1017; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1] 1018; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 1019; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 1020; GFX9-W64-NEXT: v_cmp_gt_u32_e32 vcc, 16, v0 1021; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 1022; GFX9-W64-NEXT: s_and_saveexec_b64 s[0:1], vcc 1023; GFX9-W64-NEXT: s_cbranch_execz .LBB20_2 1024; GFX9-W64-NEXT: ; %bb.1: ; %if 1025; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec 1026; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1027; GFX9-W64-NEXT: global_load_dword v2, v[3:4], off glc 1028; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1029; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v2 1030; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 1031; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 1032; GFX9-W64-NEXT: .LBB20_2: ; %endif 1033; GFX9-W64-NEXT: s_or_b64 exec, exec, s[0:1] 1034; GFX9-W64-NEXT: ; return to shader part epilog 1035; 1036; GFX10-W32-LABEL: test_strict_wqm6_then: 1037; GFX10-W32: ; %bb.0: ; %main_body 1038; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo 1039; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1040; GFX10-W32-NEXT: global_load_dword v1, v[3:4], off glc dlc 1041; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1042; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 1043; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 1044; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 1045; GFX10-W32-NEXT: v_cmp_gt_u32_e32 vcc_lo, 16, v0 1046; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 1047; GFX10-W32-NEXT: s_and_saveexec_b32 s0, vcc_lo 1048; GFX10-W32-NEXT: s_cbranch_execz .LBB20_2 1049; GFX10-W32-NEXT: ; %bb.1: ; %if 1050; GFX10-W32-NEXT: s_mov_b32 s1, exec_lo 1051; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1052; GFX10-W32-NEXT: global_load_dword v2, v[3:4], off glc dlc 1053; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1054; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v2 1055; GFX10-W32-NEXT: s_mov_b32 exec_lo, s1 1056; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 1057; GFX10-W32-NEXT: .LBB20_2: ; %endif 1058; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s0 1059; GFX10-W32-NEXT: ; return to shader part epilog 1060main_body: 1061 %src0 = load volatile float, float addrspace(1)* undef 1062 ; use mbcnt to make sure the branch is divergent 1063 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 1064 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 1065 %cc = icmp uge i32 %hi, 16 1066 br i1 %cc, label %endif, label %if 1067 1068if: 1069 %src1 = load volatile float, float addrspace(1)* undef 1070 %out = fadd float %src0, %src1 1071 %out.0 = call float @llvm.amdgcn.strict.wqm.f32(float %out) 1072 br label %endif 1073 1074endif: 1075 %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ] 1076 ret float %out.1 1077} 1078 1079; Check that Strict WQM is turned on correctly across basic block boundaries. 1080; loop version 1081;SI-CHECK: buffer_load_dword 1082;VI-CHECK: flat_load_dword 1083;SI-CHECK: buffer_load_dword 1084;VI-CHECK: flat_load_dword 1085define amdgpu_ps float @test_strict_wqm6_loop() { 1086; GFX9-W64-LABEL: test_strict_wqm6_loop: 1087; GFX9-W64: ; %bb.0: ; %main_body 1088; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec 1089; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1090; GFX9-W64-NEXT: global_load_dword v1, v[3:4], off glc 1091; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1092; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1] 1093; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 1094; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v0 1095; GFX9-W64-NEXT: s_mov_b64 s[0:1], 0 1096; GFX9-W64-NEXT: .LBB21_1: ; %loop 1097; GFX9-W64-NEXT: ; =>This Inner Loop Header: Depth=1 1098; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec 1099; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1100; GFX9-W64-NEXT: global_load_dword v2, v[3:4], off glc 1101; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1102; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 1103; GFX9-W64-NEXT: v_add_u32_e32 v3, -1, v3 1104; GFX9-W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3 1105; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec 1106; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1107; GFX9-W64-NEXT: v_add_f32_e32 v2, v1, v2 1108; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 1109; GFX9-W64-NEXT: s_or_b64 s[0:1], vcc, s[0:1] 1110; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2 1111; GFX9-W64-NEXT: s_andn2_b64 exec, exec, s[0:1] 1112; GFX9-W64-NEXT: s_cbranch_execnz .LBB21_1 1113; GFX9-W64-NEXT: ; %bb.2: ; %endloop 1114; GFX9-W64-NEXT: s_or_b64 exec, exec, s[0:1] 1115; GFX9-W64-NEXT: ; return to shader part epilog 1116; 1117; GFX10-W32-LABEL: test_strict_wqm6_loop: 1118; GFX10-W32: ; %bb.0: ; %main_body 1119; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo 1120; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1121; GFX10-W32-NEXT: global_load_dword v1, v[3:4], off glc dlc 1122; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1123; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 1124; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 1125; GFX10-W32-NEXT: s_mov_b32 s0, 0 1126; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v0 1127; GFX10-W32-NEXT: .LBB21_1: ; %loop 1128; GFX10-W32-NEXT: ; =>This Inner Loop Header: Depth=1 1129; GFX10-W32-NEXT: s_mov_b32 s1, exec_lo 1130; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1131; GFX10-W32-NEXT: global_load_dword v2, v[3:4], off glc dlc 1132; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1133; GFX10-W32-NEXT: s_mov_b32 exec_lo, s1 1134; GFX10-W32-NEXT: v_add_nc_u32_e32 v3, -1, v3 1135; GFX10-W32-NEXT: s_mov_b32 s1, exec_lo 1136; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1137; GFX10-W32-NEXT: v_add_f32_e32 v2, v1, v2 1138; GFX10-W32-NEXT: s_mov_b32 exec_lo, s1 1139; GFX10-W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v3 1140; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2 1141; GFX10-W32-NEXT: s_or_b32 s0, vcc_lo, s0 1142; GFX10-W32-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 1143; GFX10-W32-NEXT: s_cbranch_execnz .LBB21_1 1144; GFX10-W32-NEXT: ; %bb.2: ; %endloop 1145; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s0 1146; GFX10-W32-NEXT: ; return to shader part epilog 1147main_body: 1148 %src0 = load volatile float, float addrspace(1)* undef 1149 ; use mbcnt to make sure the branch is divergent 1150 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 1151 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 1152 br label %loop 1153 1154loop: 1155 %counter = phi i32 [ %hi, %main_body ], [ %counter.1, %loop ] 1156 %src1 = load volatile float, float addrspace(1)* undef 1157 %out = fadd float %src0, %src1 1158 %out.0 = call float @llvm.amdgcn.strict.wqm.f32(float %out) 1159 %counter.1 = sub i32 %counter, 1 1160 %cc = icmp ne i32 %counter.1, 0 1161 br i1 %cc, label %loop, label %endloop 1162 1163endloop: 1164 ret float %out.0 1165} 1166 1167; Check that enabling WQM anywhere enables WQM for the set.inactive source. 1168define amdgpu_ps void @test_set_inactive2(i32 inreg %idx0, i32 inreg %idx1) { 1169; GFX9-W64-LABEL: test_set_inactive2: 1170; GFX9-W64: ; %bb.0: ; %main_body 1171; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec 1172; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1173; GFX9-W64-NEXT: v_mov_b32_e32 v0, s1 1174; GFX9-W64-NEXT: v_mov_b32_e32 v2, s0 1175; GFX9-W64-NEXT: buffer_load_dword v1, v0, s[0:3], 0 idxen 1176; GFX9-W64-NEXT: s_nop 0 1177; GFX9-W64-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen 1178; GFX9-W64-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $scc killed $exec 1179; GFX9-W64-NEXT: ; kill: def $vgpr2 killed $vgpr2 killed $exec killed $exec 1180; GFX9-W64-NEXT: s_and_b64 exec, exec, s[2:3] 1181; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1182; GFX9-W64-NEXT: v_add_u32_e32 v1, v2, v1 1183; GFX9-W64-NEXT: buffer_store_dword v1, v0, s[0:3], 0 idxen 1184; GFX9-W64-NEXT: s_endpgm 1185; 1186; GFX10-W32-LABEL: test_set_inactive2: 1187; GFX10-W32: ; %bb.0: ; %main_body 1188; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo 1189; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1190; GFX10-W32-NEXT: v_mov_b32_e32 v0, s1 1191; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 1192; GFX10-W32-NEXT: s_clause 0x1 1193; GFX10-W32-NEXT: buffer_load_dword v2, v0, s[0:3], 0 idxen 1194; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 1195; GFX10-W32-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $scc killed $exec 1196; GFX10-W32-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $exec killed $exec 1197; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s2 1198; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1199; GFX10-W32-NEXT: v_add_nc_u32_e32 v1, v1, v2 1200; GFX10-W32-NEXT: buffer_store_dword v1, v0, s[0:3], 0 idxen 1201; GFX10-W32-NEXT: s_endpgm 1202main_body: 1203 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 1204 %src1.0 = bitcast float %src1 to i32 1205 %src1.1 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %src1.0, i32 undef) 1206 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 1207 %src0.0 = bitcast float %src0 to i32 1208 %src0.1 = call i32 @llvm.amdgcn.wqm.i32(i32 %src0.0) 1209 %out = add i32 %src0.1, %src1.1 1210 %out.0 = bitcast i32 %out to float 1211 call void @llvm.amdgcn.struct.buffer.store.f32(float %out.0, <4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 1212 ret void 1213} 1214 1215; Check a case of one branch of an if-else requiring WQM, the other requiring 1216; exact. 1217; Note: In this particular case, the save-and-restore could be avoided if the 1218; analysis understood that the two branches of the if-else are mutually 1219; exclusive. 1220define amdgpu_ps float @test_control_flow_0(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data) { 1221; GFX9-W64-LABEL: test_control_flow_0: 1222; GFX9-W64: ; %bb.0: ; %main_body 1223; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec 1224; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1225; GFX9-W64-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1 1226; GFX9-W64-NEXT: s_and_saveexec_b64 s[14:15], vcc 1227; GFX9-W64-NEXT: s_xor_b64 s[14:15], exec, s[14:15] 1228; GFX9-W64-NEXT: s_cbranch_execz .LBB23_2 1229; GFX9-W64-NEXT: ; %bb.1: ; %ELSE 1230; GFX9-W64-NEXT: s_and_saveexec_b64 s[16:17], s[12:13] 1231; GFX9-W64-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen 1232; GFX9-W64-NEXT: ; implicit-def: $vgpr0 1233; GFX9-W64-NEXT: s_mov_b64 exec, s[16:17] 1234; GFX9-W64-NEXT: .LBB23_2: ; %Flow 1235; GFX9-W64-NEXT: s_or_saveexec_b64 s[14:15], s[14:15] 1236; GFX9-W64-NEXT: s_xor_b64 exec, exec, s[14:15] 1237; GFX9-W64-NEXT: s_cbranch_execz .LBB23_4 1238; GFX9-W64-NEXT: ; %bb.3: ; %IF 1239; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 1240; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1241; GFX9-W64-NEXT: image_sample v2, v0, s[0:7], s[8:11] dmask:0x1 1242; GFX9-W64-NEXT: .LBB23_4: ; %END 1243; GFX9-W64-NEXT: s_or_b64 exec, exec, s[14:15] 1244; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 1245; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1246; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2 1247; GFX9-W64-NEXT: ; return to shader part epilog 1248; 1249; GFX10-W32-LABEL: test_control_flow_0: 1250; GFX10-W32: ; %bb.0: ; %main_body 1251; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo 1252; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1253; GFX10-W32-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1 1254; GFX10-W32-NEXT: s_and_saveexec_b32 s13, vcc_lo 1255; GFX10-W32-NEXT: s_xor_b32 s13, exec_lo, s13 1256; GFX10-W32-NEXT: s_cbranch_execz .LBB23_2 1257; GFX10-W32-NEXT: ; %bb.1: ; %ELSE 1258; GFX10-W32-NEXT: s_and_saveexec_b32 s14, s12 1259; GFX10-W32-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen 1260; GFX10-W32-NEXT: ; implicit-def: $vgpr0 1261; GFX10-W32-NEXT: s_mov_b32 exec_lo, s14 1262; GFX10-W32-NEXT: .LBB23_2: ; %Flow 1263; GFX10-W32-NEXT: s_or_saveexec_b32 s13, s13 1264; GFX10-W32-NEXT: s_xor_b32 exec_lo, exec_lo, s13 1265; GFX10-W32-NEXT: s_cbranch_execz .LBB23_4 1266; GFX10-W32-NEXT: ; %bb.3: ; %IF 1267; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 1268; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1269; GFX10-W32-NEXT: image_sample v2, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 1270; GFX10-W32-NEXT: .LBB23_4: ; %END 1271; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s13 1272; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 1273; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1274; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2 1275; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 1276; GFX10-W32-NEXT: ; return to shader part epilog 1277main_body: 1278 %cmp = icmp eq i32 %z, 0 1279 br i1 %cmp, label %IF, label %ELSE 1280 1281IF: 1282 %c.bc = bitcast i32 %c to float 1283 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1284 %tex0 = extractelement <4 x float> %tex, i32 0 1285 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1286 %data.if = extractelement <4 x float> %dtex, i32 0 1287 br label %END 1288 1289ELSE: 1290 call void @llvm.amdgcn.struct.buffer.store.f32(float %data, <4 x i32> undef, i32 %c, i32 0, i32 0, i32 0) 1291 br label %END 1292 1293END: 1294 %r = phi float [ %data.if, %IF ], [ %data, %ELSE ] 1295 ret float %r 1296} 1297 1298; Reverse branch order compared to the previous test. 1299define amdgpu_ps float @test_control_flow_1(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data) { 1300; GFX9-W64-LABEL: test_control_flow_1: 1301; GFX9-W64: ; %bb.0: ; %main_body 1302; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec 1303; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1304; GFX9-W64-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1 1305; GFX9-W64-NEXT: s_and_saveexec_b64 s[14:15], vcc 1306; GFX9-W64-NEXT: s_xor_b64 s[14:15], exec, s[14:15] 1307; GFX9-W64-NEXT: s_cbranch_execz .LBB24_2 1308; GFX9-W64-NEXT: ; %bb.1: ; %IF 1309; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 1310; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1311; GFX9-W64-NEXT: image_sample v2, v0, s[0:7], s[8:11] dmask:0x1 1312; GFX9-W64-NEXT: ; implicit-def: $vgpr0 1313; GFX9-W64-NEXT: .LBB24_2: ; %Flow 1314; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], s[14:15] 1315; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 1316; GFX9-W64-NEXT: s_and_b64 s[0:1], exec, s[0:1] 1317; GFX9-W64-NEXT: s_xor_b64 exec, exec, s[0:1] 1318; GFX9-W64-NEXT: s_cbranch_execz .LBB24_4 1319; GFX9-W64-NEXT: ; %bb.3: ; %ELSE 1320; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1321; GFX9-W64-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen 1322; GFX9-W64-NEXT: .LBB24_4: ; %END 1323; GFX9-W64-NEXT: s_or_b64 exec, exec, s[0:1] 1324; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1325; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2 1326; GFX9-W64-NEXT: ; return to shader part epilog 1327; 1328; GFX10-W32-LABEL: test_control_flow_1: 1329; GFX10-W32: ; %bb.0: ; %main_body 1330; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo 1331; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1332; GFX10-W32-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1 1333; GFX10-W32-NEXT: s_and_saveexec_b32 s13, vcc_lo 1334; GFX10-W32-NEXT: s_xor_b32 s13, exec_lo, s13 1335; GFX10-W32-NEXT: s_cbranch_execz .LBB24_2 1336; GFX10-W32-NEXT: ; %bb.1: ; %IF 1337; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 1338; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1339; GFX10-W32-NEXT: image_sample v2, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 1340; GFX10-W32-NEXT: ; implicit-def: $vgpr0 1341; GFX10-W32-NEXT: .LBB24_2: ; %Flow 1342; GFX10-W32-NEXT: s_or_saveexec_b32 s0, s13 1343; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 1344; GFX10-W32-NEXT: s_and_b32 s0, exec_lo, s0 1345; GFX10-W32-NEXT: s_xor_b32 exec_lo, exec_lo, s0 1346; GFX10-W32-NEXT: s_cbranch_execz .LBB24_4 1347; GFX10-W32-NEXT: ; %bb.3: ; %ELSE 1348; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1349; GFX10-W32-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen 1350; GFX10-W32-NEXT: .LBB24_4: ; %END 1351; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s0 1352; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1353; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2 1354; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 1355; GFX10-W32-NEXT: ; return to shader part epilog 1356main_body: 1357 %cmp = icmp eq i32 %z, 0 1358 br i1 %cmp, label %ELSE, label %IF 1359 1360IF: 1361 %c.bc = bitcast i32 %c to float 1362 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1363 %tex0 = extractelement <4 x float> %tex, i32 0 1364 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1365 %data.if = extractelement <4 x float> %dtex, i32 0 1366 br label %END 1367 1368ELSE: 1369 call void @llvm.amdgcn.struct.buffer.store.f32(float %data, <4 x i32> undef, i32 %c, i32 0, i32 0, i32 0) 1370 br label %END 1371 1372END: 1373 %r = phi float [ %data.if, %IF ], [ %data, %ELSE ] 1374 ret float %r 1375} 1376 1377; Check that branch conditions are properly marked as needing WQM... 1378define amdgpu_ps <4 x float> @test_control_flow_2(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, <3 x i32> %idx, <2 x float> %data, i32 %coord) { 1379; GFX9-W64-LABEL: test_control_flow_2: 1380; GFX9-W64: ; %bb.0: ; %main_body 1381; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec 1382; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1383; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 1384; GFX9-W64-NEXT: buffer_store_dword v3, v0, s[0:3], 0 idxen 1385; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1386; GFX9-W64-NEXT: buffer_load_dword v0, v1, s[0:3], 0 idxen 1387; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 1388; GFX9-W64-NEXT: buffer_store_dword v4, v2, s[0:3], 0 idxen 1389; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1390; GFX9-W64-NEXT: s_waitcnt vmcnt(1) 1391; GFX9-W64-NEXT: v_cmp_nlt_f32_e32 vcc, 0, v0 1392; GFX9-W64-NEXT: ; implicit-def: $vgpr0 1393; GFX9-W64-NEXT: s_and_saveexec_b64 s[14:15], vcc 1394; GFX9-W64-NEXT: s_xor_b64 s[14:15], exec, s[14:15] 1395; GFX9-W64-NEXT: ; %bb.1: ; %ELSE 1396; GFX9-W64-NEXT: v_lshlrev_b32_e32 v0, 2, v5 1397; GFX9-W64-NEXT: ; implicit-def: $vgpr5 1398; GFX9-W64-NEXT: ; %bb.2: ; %Flow 1399; GFX9-W64-NEXT: s_or_saveexec_b64 s[14:15], s[14:15] 1400; GFX9-W64-NEXT: s_xor_b64 exec, exec, s[14:15] 1401; GFX9-W64-NEXT: ; %bb.3: ; %IF 1402; GFX9-W64-NEXT: v_mul_lo_u32 v0, v5, 3 1403; GFX9-W64-NEXT: ; %bb.4: ; %END 1404; GFX9-W64-NEXT: s_or_b64 exec, exec, s[14:15] 1405; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 1406; GFX9-W64-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf 1407; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1408; GFX9-W64-NEXT: ; return to shader part epilog 1409; 1410; GFX10-W32-LABEL: test_control_flow_2: 1411; GFX10-W32: ; %bb.0: ; %main_body 1412; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo 1413; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1414; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 1415; GFX10-W32-NEXT: buffer_store_dword v3, v0, s[0:3], 0 idxen 1416; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1417; GFX10-W32-NEXT: buffer_load_dword v0, v1, s[0:3], 0 idxen 1418; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1419; GFX10-W32-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0, v0 1420; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 1421; GFX10-W32-NEXT: buffer_store_dword v4, v2, s[0:3], 0 idxen 1422; GFX10-W32-NEXT: ; implicit-def: $vgpr0 1423; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1424; GFX10-W32-NEXT: s_and_saveexec_b32 s13, vcc_lo 1425; GFX10-W32-NEXT: s_xor_b32 s13, exec_lo, s13 1426; GFX10-W32-NEXT: ; %bb.1: ; %ELSE 1427; GFX10-W32-NEXT: v_lshlrev_b32_e32 v0, 2, v5 1428; GFX10-W32-NEXT: ; implicit-def: $vgpr5 1429; GFX10-W32-NEXT: ; %bb.2: ; %Flow 1430; GFX10-W32-NEXT: s_or_saveexec_b32 s13, s13 1431; GFX10-W32-NEXT: s_xor_b32 exec_lo, exec_lo, s13 1432; GFX10-W32-NEXT: ; %bb.3: ; %IF 1433; GFX10-W32-NEXT: v_mul_lo_u32 v0, v5, 3 1434; GFX10-W32-NEXT: ; %bb.4: ; %END 1435; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s13 1436; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 1437; GFX10-W32-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D 1438; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1439; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 1440; GFX10-W32-NEXT: ; return to shader part epilog 1441main_body: 1442 %idx.1 = extractelement <3 x i32> %idx, i32 0 1443 %data.1 = extractelement <2 x float> %data, i32 0 1444 call void @llvm.amdgcn.struct.buffer.store.f32(float %data.1, <4 x i32> undef, i32 %idx.1, i32 0, i32 0, i32 0) 1445 1446 ; The load that determines the branch (and should therefore be WQM) is 1447 ; surrounded by stores that require disabled WQM. 1448 %idx.2 = extractelement <3 x i32> %idx, i32 1 1449 %z = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx.2, i32 0, i32 0, i32 0) 1450 1451 %idx.3 = extractelement <3 x i32> %idx, i32 2 1452 %data.3 = extractelement <2 x float> %data, i32 1 1453 call void @llvm.amdgcn.struct.buffer.store.f32(float %data.3, <4 x i32> undef, i32 %idx.3, i32 0, i32 0, i32 0) 1454 1455 %cc = fcmp ogt float %z, 0.0 1456 br i1 %cc, label %IF, label %ELSE 1457 1458IF: 1459 %coord.IF = mul i32 %coord, 3 1460 br label %END 1461 1462ELSE: 1463 %coord.ELSE = mul i32 %coord, 4 1464 br label %END 1465 1466END: 1467 %coord.END = phi i32 [ %coord.IF, %IF ], [ %coord.ELSE, %ELSE ] 1468 %coord.END.bc = bitcast i32 %coord.END to float 1469 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord.END.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1470 ret <4 x float> %tex 1471} 1472 1473; ... but only if they really do need it. 1474define amdgpu_ps float @test_control_flow_3(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %idx, float %coord) { 1475; GFX9-W64-LABEL: test_control_flow_3: 1476; GFX9-W64: ; %bb.0: ; %main_body 1477; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec 1478; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1479; GFX9-W64-NEXT: image_sample v1, v1, s[0:7], s[8:11] dmask:0x1 1480; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 1481; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1482; GFX9-W64-NEXT: image_sample v1, v1, s[0:7], s[8:11] dmask:0x1 1483; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1484; GFX9-W64-NEXT: v_cmp_nlt_f32_e32 vcc, 0, v1 1485; GFX9-W64-NEXT: buffer_store_dword v1, v0, s[0:3], 0 idxen 1486; GFX9-W64-NEXT: ; implicit-def: $vgpr0 1487; GFX9-W64-NEXT: s_and_saveexec_b64 s[0:1], vcc 1488; GFX9-W64-NEXT: s_xor_b64 s[0:1], exec, s[0:1] 1489; GFX9-W64-NEXT: ; %bb.1: ; %ELSE 1490; GFX9-W64-NEXT: v_mul_f32_e32 v0, 4.0, v1 1491; GFX9-W64-NEXT: ; implicit-def: $vgpr1 1492; GFX9-W64-NEXT: ; %bb.2: ; %Flow 1493; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], s[0:1] 1494; GFX9-W64-NEXT: s_xor_b64 exec, exec, s[0:1] 1495; GFX9-W64-NEXT: ; %bb.3: ; %IF 1496; GFX9-W64-NEXT: v_mul_f32_e32 v0, 0x40400000, v1 1497; GFX9-W64-NEXT: ; %bb.4: ; %END 1498; GFX9-W64-NEXT: s_or_b64 exec, exec, s[0:1] 1499; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1500; GFX9-W64-NEXT: ; return to shader part epilog 1501; 1502; GFX10-W32-LABEL: test_control_flow_3: 1503; GFX10-W32: ; %bb.0: ; %main_body 1504; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo 1505; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1506; GFX10-W32-NEXT: image_sample v1, v1, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 1507; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 1508; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1509; GFX10-W32-NEXT: image_sample v1, v1, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 1510; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1511; GFX10-W32-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0, v1 1512; GFX10-W32-NEXT: buffer_store_dword v1, v0, s[0:3], 0 idxen 1513; GFX10-W32-NEXT: ; implicit-def: $vgpr0 1514; GFX10-W32-NEXT: s_and_saveexec_b32 s0, vcc_lo 1515; GFX10-W32-NEXT: s_xor_b32 s0, exec_lo, s0 1516; GFX10-W32-NEXT: ; %bb.1: ; %ELSE 1517; GFX10-W32-NEXT: v_mul_f32_e32 v0, 4.0, v1 1518; GFX10-W32-NEXT: ; implicit-def: $vgpr1 1519; GFX10-W32-NEXT: ; %bb.2: ; %Flow 1520; GFX10-W32-NEXT: s_or_saveexec_b32 s0, s0 1521; GFX10-W32-NEXT: s_xor_b32 exec_lo, exec_lo, s0 1522; GFX10-W32-NEXT: ; %bb.3: ; %IF 1523; GFX10-W32-NEXT: v_mul_f32_e32 v0, 0x40400000, v1 1524; GFX10-W32-NEXT: ; %bb.4: ; %END 1525; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s0 1526; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 1527; GFX10-W32-NEXT: ; return to shader part epilog 1528main_body: 1529 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1530 %tex0 = extractelement <4 x float> %tex, i32 0 1531 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1532 %dtex.1 = extractelement <4 x float> %dtex, i32 0 1533 call void @llvm.amdgcn.struct.buffer.store.f32(float %dtex.1, <4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 1534 1535 %cc = fcmp ogt float %dtex.1, 0.0 1536 br i1 %cc, label %IF, label %ELSE 1537 1538IF: 1539 %tex.IF = fmul float %dtex.1, 3.0 1540 br label %END 1541 1542ELSE: 1543 %tex.ELSE = fmul float %dtex.1, 4.0 1544 br label %END 1545 1546END: 1547 %tex.END = phi float [ %tex.IF, %IF ], [ %tex.ELSE, %ELSE ] 1548 ret float %tex.END 1549} 1550 1551; Another test that failed at some point because of terminator handling. 1552define amdgpu_ps <4 x float> @test_control_flow_4(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float %coord, i32 %y, float %z) { 1553; GFX9-W64-LABEL: test_control_flow_4: 1554; GFX9-W64: ; %bb.0: ; %main_body 1555; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec 1556; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1557; GFX9-W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 1558; GFX9-W64-NEXT: s_and_saveexec_b64 s[14:15], vcc 1559; GFX9-W64-NEXT: s_cbranch_execz .LBB27_2 1560; GFX9-W64-NEXT: ; %bb.1: ; %IF 1561; GFX9-W64-NEXT: s_and_saveexec_b64 s[16:17], s[12:13] 1562; GFX9-W64-NEXT: buffer_load_dword v1, off, s[0:3], 0 1563; GFX9-W64-NEXT: v_mov_b32_e32 v2, 1 1564; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1565; GFX9-W64-NEXT: buffer_store_dword v1, v2, s[0:3], 0 idxen 1566; GFX9-W64-NEXT: s_mov_b64 exec, s[16:17] 1567; GFX9-W64-NEXT: .LBB27_2: ; %END 1568; GFX9-W64-NEXT: s_or_b64 exec, exec, s[14:15] 1569; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 1570; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 1571; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1572; GFX9-W64-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf 1573; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1574; GFX9-W64-NEXT: ; return to shader part epilog 1575; 1576; GFX10-W32-LABEL: test_control_flow_4: 1577; GFX10-W32: ; %bb.0: ; %main_body 1578; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo 1579; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1580; GFX10-W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1 1581; GFX10-W32-NEXT: s_and_saveexec_b32 s13, vcc_lo 1582; GFX10-W32-NEXT: s_cbranch_execz .LBB27_2 1583; GFX10-W32-NEXT: ; %bb.1: ; %IF 1584; GFX10-W32-NEXT: s_and_saveexec_b32 s14, s12 1585; GFX10-W32-NEXT: buffer_load_dword v1, off, s[0:3], 0 1586; GFX10-W32-NEXT: v_mov_b32_e32 v2, 1 1587; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1588; GFX10-W32-NEXT: buffer_store_dword v1, v2, s[0:3], 0 idxen 1589; GFX10-W32-NEXT: s_mov_b32 exec_lo, s14 1590; GFX10-W32-NEXT: .LBB27_2: ; %END 1591; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s13 1592; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 1593; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 1594; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1595; GFX10-W32-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D 1596; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1597; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 1598; GFX10-W32-NEXT: ; return to shader part epilog 1599main_body: 1600 %cond = icmp eq i32 %y, 0 1601 br i1 %cond, label %IF, label %END 1602 1603IF: 1604 %data = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> undef, i32 0, i32 0, i32 0) 1605 call void @llvm.amdgcn.struct.buffer.store.f32(float %data, <4 x i32> undef, i32 1, i32 0, i32 0, i32 0) 1606 br label %END 1607 1608END: 1609 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1610 %tex0 = extractelement <4 x float> %tex, i32 0 1611 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1612 ret <4 x float> %dtex 1613} 1614 1615; Kill is performed in WQM mode so that uniform kill behaves correctly ... 1616define amdgpu_ps <4 x float> @test_kill_0(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float addrspace(1)* inreg %ptr, <2 x i32> %idx, <2 x float> %data, float %coord, float %coord2, float %z) { 1617; GFX9-W64-LABEL: test_kill_0: 1618; GFX9-W64: ; %bb.0: ; %main_body 1619; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec 1620; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1621; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 1622; GFX9-W64-NEXT: image_sample v[7:10], v4, s[0:7], s[8:11] dmask:0xf 1623; GFX9-W64-NEXT: s_nop 0 1624; GFX9-W64-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen 1625; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1626; GFX9-W64-NEXT: v_cmp_ngt_f32_e32 vcc, 0, v6 1627; GFX9-W64-NEXT: s_andn2_b64 s[12:13], s[12:13], vcc 1628; GFX9-W64-NEXT: s_cbranch_scc0 .LBB28_2 1629; GFX9-W64-NEXT: ; %bb.1: ; %main_body 1630; GFX9-W64-NEXT: s_andn2_b64 exec, exec, vcc 1631; GFX9-W64-NEXT: image_sample v0, v5, s[0:7], s[8:11] dmask:0x1 1632; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 1633; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1634; GFX9-W64-NEXT: image_sample v[11:14], v0, s[0:7], s[8:11] dmask:0xf 1635; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1636; GFX9-W64-NEXT: v_add_f32_e32 v0, v7, v11 1637; GFX9-W64-NEXT: buffer_store_dword v3, v1, s[0:3], 0 idxen 1638; GFX9-W64-NEXT: v_add_f32_e32 v1, v8, v12 1639; GFX9-W64-NEXT: v_add_f32_e32 v2, v9, v13 1640; GFX9-W64-NEXT: v_add_f32_e32 v3, v10, v14 1641; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1642; GFX9-W64-NEXT: s_branch .LBB28_3 1643; GFX9-W64-NEXT: .LBB28_2: 1644; GFX9-W64-NEXT: s_mov_b64 exec, 0 1645; GFX9-W64-NEXT: exp null off, off, off, off done vm 1646; GFX9-W64-NEXT: s_endpgm 1647; GFX9-W64-NEXT: .LBB28_3: 1648; 1649; GFX10-W32-LABEL: test_kill_0: 1650; GFX10-W32: ; %bb.0: ; %main_body 1651; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo 1652; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1653; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 1654; GFX10-W32-NEXT: image_sample v[7:10], v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D 1655; GFX10-W32-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen 1656; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1657; GFX10-W32-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0, v6 1658; GFX10-W32-NEXT: s_andn2_b32 s12, s12, vcc_lo 1659; GFX10-W32-NEXT: s_cbranch_scc0 .LBB28_2 1660; GFX10-W32-NEXT: ; %bb.1: ; %main_body 1661; GFX10-W32-NEXT: s_andn2_b32 exec_lo, exec_lo, vcc_lo 1662; GFX10-W32-NEXT: image_sample v0, v5, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 1663; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 1664; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1665; GFX10-W32-NEXT: image_sample v[11:14], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D 1666; GFX10-W32-NEXT: buffer_store_dword v3, v1, s[0:3], 0 idxen 1667; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1668; GFX10-W32-NEXT: v_add_f32_e32 v4, v8, v12 1669; GFX10-W32-NEXT: v_add_f32_e32 v5, v10, v14 1670; GFX10-W32-NEXT: v_add_f32_e32 v0, v7, v11 1671; GFX10-W32-NEXT: v_add_f32_e32 v2, v9, v13 1672; GFX10-W32-NEXT: v_mov_b32_e32 v1, v4 1673; GFX10-W32-NEXT: v_mov_b32_e32 v3, v5 1674; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 1675; GFX10-W32-NEXT: s_branch .LBB28_3 1676; GFX10-W32-NEXT: .LBB28_2: 1677; GFX10-W32-NEXT: s_mov_b32 exec_lo, 0 1678; GFX10-W32-NEXT: exp null off, off, off, off done vm 1679; GFX10-W32-NEXT: s_endpgm 1680; GFX10-W32-NEXT: .LBB28_3: 1681main_body: 1682 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1683 %idx.0 = extractelement <2 x i32> %idx, i32 0 1684 %data.0 = extractelement <2 x float> %data, i32 0 1685 call void @llvm.amdgcn.struct.buffer.store.f32(float %data.0, <4 x i32> undef, i32 %idx.0, i32 0, i32 0, i32 0) 1686 1687 %z.cmp = fcmp olt float %z, 0.0 1688 call void @llvm.amdgcn.kill(i1 %z.cmp) 1689 1690 %idx.1 = extractelement <2 x i32> %idx, i32 1 1691 %data.1 = extractelement <2 x float> %data, i32 1 1692 call void @llvm.amdgcn.struct.buffer.store.f32(float %data.1, <4 x i32> undef, i32 %idx.1, i32 0, i32 0, i32 0) 1693 %tex2 = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord2, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1694 %tex2.0 = extractelement <4 x float> %tex2, i32 0 1695 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex2.0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1696 %out = fadd <4 x float> %tex, %dtex 1697 1698 ret <4 x float> %out 1699} 1700 1701; ... but only if WQM is necessary. 1702; CHECK-LABEL: {{^}}test_kill_1: 1703; CHECK-NEXT: ; %main_body 1704; CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 1705; CHECK: s_wqm_b64 exec, exec 1706; CHECK: image_sample 1707; CHECK: s_and_b64 exec, exec, [[ORIG]] 1708; CHECK: image_sample 1709; CHECK-NOT: wqm 1710; CHECK-DAG: buffer_store_dword 1711; CHECK-DAG: v_cmp_ 1712define amdgpu_ps <4 x float> @test_kill_1(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %idx, float %data, float %coord, float %coord2, float %z) { 1713; GFX9-W64-LABEL: test_kill_1: 1714; GFX9-W64: ; %bb.0: ; %main_body 1715; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec 1716; GFX9-W64-NEXT: v_mov_b32_e32 v4, v2 1717; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1718; GFX9-W64-NEXT: v_mov_b32_e32 v5, v0 1719; GFX9-W64-NEXT: image_sample v0, v1, s[0:7], s[8:11] dmask:0x1 1720; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 1721; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1722; GFX9-W64-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf 1723; GFX9-W64-NEXT: v_cmp_ngt_f32_e32 vcc, 0, v4 1724; GFX9-W64-NEXT: s_andn2_b64 s[12:13], s[12:13], vcc 1725; GFX9-W64-NEXT: buffer_store_dword v5, off, s[0:3], 0 1726; GFX9-W64-NEXT: s_cbranch_scc0 .LBB29_2 1727; GFX9-W64-NEXT: ; %bb.1: ; %main_body 1728; GFX9-W64-NEXT: s_andn2_b64 exec, exec, vcc 1729; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1730; GFX9-W64-NEXT: s_branch .LBB29_3 1731; GFX9-W64-NEXT: .LBB29_2: 1732; GFX9-W64-NEXT: s_mov_b64 exec, 0 1733; GFX9-W64-NEXT: exp null off, off, off, off done vm 1734; GFX9-W64-NEXT: s_endpgm 1735; GFX9-W64-NEXT: .LBB29_3: 1736; 1737; GFX10-W32-LABEL: test_kill_1: 1738; GFX10-W32: ; %bb.0: ; %main_body 1739; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo 1740; GFX10-W32-NEXT: v_mov_b32_e32 v4, v2 1741; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1742; GFX10-W32-NEXT: v_mov_b32_e32 v5, v0 1743; GFX10-W32-NEXT: image_sample v0, v1, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 1744; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 1745; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1746; GFX10-W32-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D 1747; GFX10-W32-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0, v4 1748; GFX10-W32-NEXT: buffer_store_dword v5, off, s[0:3], 0 1749; GFX10-W32-NEXT: s_andn2_b32 s12, s12, vcc_lo 1750; GFX10-W32-NEXT: s_cbranch_scc0 .LBB29_2 1751; GFX10-W32-NEXT: ; %bb.1: ; %main_body 1752; GFX10-W32-NEXT: s_andn2_b32 exec_lo, exec_lo, vcc_lo 1753; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1754; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 1755; GFX10-W32-NEXT: s_branch .LBB29_3 1756; GFX10-W32-NEXT: .LBB29_2: 1757; GFX10-W32-NEXT: s_mov_b32 exec_lo, 0 1758; GFX10-W32-NEXT: exp null off, off, off, off done vm 1759; GFX10-W32-NEXT: s_endpgm 1760; GFX10-W32-NEXT: .LBB29_3: 1761main_body: 1762 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1763 %tex0 = extractelement <4 x float> %tex, i32 0 1764 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1765 1766 call void @llvm.amdgcn.raw.buffer.store.f32(float %data, <4 x i32> undef, i32 0, i32 0, i32 0) 1767 1768 %z.cmp = fcmp olt float %z, 0.0 1769 call void @llvm.amdgcn.kill(i1 %z.cmp) 1770 1771 ret <4 x float> %dtex 1772} 1773 1774; Check prolog shaders. 1775; CHECK-LABEL: {{^}}test_prolog_1: 1776; CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 1777; CHECK: s_wqm_b64 exec, exec 1778; CHECK: v_add_f32_e32 v0, 1779; CHECK: s_and_b64 exec, exec, [[ORIG]] 1780define amdgpu_ps float @test_prolog_1(float %a, float %b) #5 { 1781; GFX9-W64-LABEL: test_prolog_1: 1782; GFX9-W64: ; %bb.0: ; %main_body 1783; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec 1784; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1785; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v1 1786; GFX9-W64-NEXT: s_and_b64 exec, exec, s[0:1] 1787; GFX9-W64-NEXT: ; return to shader part epilog 1788; 1789; GFX10-W32-LABEL: test_prolog_1: 1790; GFX10-W32: ; %bb.0: ; %main_body 1791; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo 1792; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1793; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v1 1794; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s0 1795; GFX10-W32-NEXT: ; return to shader part epilog 1796main_body: 1797 %s = fadd float %a, %b 1798 ret float %s 1799} 1800 1801; CHECK-LABEL: {{^}}test_loop_vcc: 1802; CHECK-NEXT: ; %entry 1803; CHECK-NEXT: s_mov_b64 [[LIVE:s\[[0-9]+:[0-9]+\]]], exec 1804; CHECK: s_wqm_b64 exec, exec 1805; CHECK: v_mov 1806; CHECK: v_mov 1807; CHECK: v_mov 1808; CHECK: v_mov 1809; CHECK: s_and_b64 exec, exec, [[LIVE]] 1810; CHECK: image_store 1811; CHECK: s_wqm_b64 exec, exec 1812; CHECK-DAG: v_mov_b32_e32 [[CTR:v[0-9]+]], 0 1813; CHECK-DAG: s_mov_b32 [[SEVEN:s[0-9]+]], 0x40e00000 1814 1815; CHECK: [[LOOPHDR:.LBB[0-9]+_[0-9]+]]: ; %body 1816; CHECK: v_add_f32_e32 [[CTR]], 2.0, [[CTR]] 1817; CHECK: [[LOOP:.LBB[0-9]+_[0-9]+]]: ; %loop 1818; CHECK: v_cmp_lt_f32_e32 vcc, [[SEVEN]], [[CTR]] 1819; CHECK: s_cbranch_vccz [[LOOPHDR]] 1820 1821; CHECK: ; %break 1822; CHECK: ; return 1823define amdgpu_ps <4 x float> @test_loop_vcc(<4 x float> %in) nounwind { 1824; GFX9-W64-LABEL: test_loop_vcc: 1825; GFX9-W64: ; %bb.0: ; %entry 1826; GFX9-W64-NEXT: s_mov_b64 s[8:9], exec 1827; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1828; GFX9-W64-NEXT: v_mov_b32_e32 v7, v3 1829; GFX9-W64-NEXT: v_mov_b32_e32 v6, v2 1830; GFX9-W64-NEXT: v_mov_b32_e32 v5, v1 1831; GFX9-W64-NEXT: v_mov_b32_e32 v4, v0 1832; GFX9-W64-NEXT: s_and_b64 exec, exec, s[8:9] 1833; GFX9-W64-NEXT: s_mov_b32 s0, 0 1834; GFX9-W64-NEXT: s_mov_b32 s1, s0 1835; GFX9-W64-NEXT: s_mov_b32 s2, s0 1836; GFX9-W64-NEXT: s_mov_b32 s3, s0 1837; GFX9-W64-NEXT: s_mov_b32 s4, s0 1838; GFX9-W64-NEXT: s_mov_b32 s5, s0 1839; GFX9-W64-NEXT: s_mov_b32 s6, s0 1840; GFX9-W64-NEXT: s_mov_b32 s7, s0 1841; GFX9-W64-NEXT: image_store v[4:7], v0, s[0:7] dmask:0xf unorm 1842; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1843; GFX9-W64-NEXT: v_mov_b32_e32 v8, 0 1844; GFX9-W64-NEXT: s_mov_b32 s10, 0x40e00000 1845; GFX9-W64-NEXT: s_branch .LBB31_2 1846; GFX9-W64-NEXT: .LBB31_1: ; %body 1847; GFX9-W64-NEXT: ; in Loop: Header=BB31_2 Depth=1 1848; GFX9-W64-NEXT: s_mov_b32 s1, s0 1849; GFX9-W64-NEXT: s_mov_b32 s2, s0 1850; GFX9-W64-NEXT: s_mov_b32 s3, s0 1851; GFX9-W64-NEXT: s_mov_b32 s4, s0 1852; GFX9-W64-NEXT: s_mov_b32 s5, s0 1853; GFX9-W64-NEXT: s_mov_b32 s6, s0 1854; GFX9-W64-NEXT: s_mov_b32 s7, s0 1855; GFX9-W64-NEXT: image_sample v[4:7], v0, s[0:7], s[0:3] dmask:0xf 1856; GFX9-W64-NEXT: v_add_f32_e32 v8, 2.0, v8 1857; GFX9-W64-NEXT: s_mov_b64 s[2:3], 0 1858; GFX9-W64-NEXT: s_cbranch_execz .LBB31_4 1859; GFX9-W64-NEXT: .LBB31_2: ; %loop 1860; GFX9-W64-NEXT: ; =>This Inner Loop Header: Depth=1 1861; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1862; GFX9-W64-NEXT: v_mov_b32_e32 v0, v4 1863; GFX9-W64-NEXT: v_cmp_lt_f32_e32 vcc, s10, v8 1864; GFX9-W64-NEXT: v_mov_b32_e32 v1, v5 1865; GFX9-W64-NEXT: v_mov_b32_e32 v2, v6 1866; GFX9-W64-NEXT: v_mov_b32_e32 v3, v7 1867; GFX9-W64-NEXT: s_cbranch_vccz .LBB31_1 1868; GFX9-W64-NEXT: ; %bb.3: 1869; GFX9-W64-NEXT: s_mov_b64 s[2:3], -1 1870; GFX9-W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7 1871; GFX9-W64-NEXT: ; implicit-def: $vgpr8 1872; GFX9-W64-NEXT: .LBB31_4: ; %break 1873; GFX9-W64-NEXT: s_and_b64 exec, exec, s[8:9] 1874; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1875; GFX9-W64-NEXT: ; return to shader part epilog 1876; 1877; GFX10-W32-LABEL: test_loop_vcc: 1878; GFX10-W32: ; %bb.0: ; %entry 1879; GFX10-W32-NEXT: s_mov_b32 s8, exec_lo 1880; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1881; GFX10-W32-NEXT: v_mov_b32_e32 v8, 0 1882; GFX10-W32-NEXT: s_mov_b32 s0, 0 1883; GFX10-W32-NEXT: s_mov_b32 s1, s0 1884; GFX10-W32-NEXT: s_mov_b32 s2, s0 1885; GFX10-W32-NEXT: s_mov_b32 s3, s0 1886; GFX10-W32-NEXT: s_mov_b32 s4, s0 1887; GFX10-W32-NEXT: s_mov_b32 s5, s0 1888; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s8 1889; GFX10-W32-NEXT: s_mov_b32 s6, s0 1890; GFX10-W32-NEXT: s_mov_b32 s7, s0 1891; GFX10-W32-NEXT: image_store v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm 1892; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 1893; GFX10-W32-NEXT: s_branch .LBB31_2 1894; GFX10-W32-NEXT: .p2align 6 1895; GFX10-W32-NEXT: .LBB31_1: ; %body 1896; GFX10-W32-NEXT: ; in Loop: Header=BB31_2 Depth=1 1897; GFX10-W32-NEXT: s_mov_b32 s1, s0 1898; GFX10-W32-NEXT: s_mov_b32 s2, s0 1899; GFX10-W32-NEXT: s_mov_b32 s3, s0 1900; GFX10-W32-NEXT: s_mov_b32 s4, s0 1901; GFX10-W32-NEXT: s_mov_b32 s5, s0 1902; GFX10-W32-NEXT: s_mov_b32 s6, s0 1903; GFX10-W32-NEXT: s_mov_b32 s7, s0 1904; GFX10-W32-NEXT: v_add_f32_e32 v8, 2.0, v8 1905; GFX10-W32-NEXT: image_sample v[0:3], v4, s[0:7], s[0:3] dmask:0xf dim:SQ_RSRC_IMG_1D 1906; GFX10-W32-NEXT: s_mov_b32 s1, 0 1907; GFX10-W32-NEXT: s_cbranch_execz .LBB31_4 1908; GFX10-W32-NEXT: .LBB31_2: ; %loop 1909; GFX10-W32-NEXT: ; =>This Inner Loop Header: Depth=1 1910; GFX10-W32-NEXT: v_cmp_lt_f32_e32 vcc_lo, 0x40e00000, v8 1911; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1912; GFX10-W32-NEXT: v_mov_b32_e32 v7, v3 1913; GFX10-W32-NEXT: v_mov_b32_e32 v6, v2 1914; GFX10-W32-NEXT: v_mov_b32_e32 v5, v1 1915; GFX10-W32-NEXT: v_mov_b32_e32 v4, v0 1916; GFX10-W32-NEXT: s_cbranch_vccz .LBB31_1 1917; GFX10-W32-NEXT: ; %bb.3: 1918; GFX10-W32-NEXT: s_mov_b32 s1, -1 1919; GFX10-W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3 1920; GFX10-W32-NEXT: ; implicit-def: $vgpr8 1921; GFX10-W32-NEXT: .LBB31_4: ; %break 1922; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s8 1923; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 1924; GFX10-W32-NEXT: v_mov_b32_e32 v0, v4 1925; GFX10-W32-NEXT: v_mov_b32_e32 v1, v5 1926; GFX10-W32-NEXT: v_mov_b32_e32 v2, v6 1927; GFX10-W32-NEXT: v_mov_b32_e32 v3, v7 1928; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 1929; GFX10-W32-NEXT: ; return to shader part epilog 1930entry: 1931 call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> %in, i32 15, i32 undef, <8 x i32> undef, i32 0, i32 0) 1932 br label %loop 1933 1934loop: 1935 %ctr.iv = phi float [ 0.0, %entry ], [ %ctr.next, %body ] 1936 %c.iv = phi <4 x float> [ %in, %entry ], [ %c.next, %body ] 1937 %cc = fcmp ogt float %ctr.iv, 7.0 1938 br i1 %cc, label %break, label %body 1939 1940body: 1941 %c.iv0 = extractelement <4 x float> %c.iv, i32 0 1942 %c.next = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.iv0, <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0) #0 1943 %ctr.next = fadd float %ctr.iv, 2.0 1944 br label %loop 1945 1946break: 1947 ret <4 x float> %c.iv 1948} 1949 1950; Only intrinsic stores need exact execution -- other stores do not have 1951; externally visible effects and may require WQM for correctness. 1952; CHECK-LABEL: {{^}}test_alloca: 1953; CHECK: s_mov_b64 [[LIVE:s\[[0-9]+:[0-9]+\]]], exec 1954; CHECK: s_wqm_b64 exec, exec 1955 1956; CHECK: s_and_b64 exec, exec, [[LIVE]] 1957; CHECK: buffer_store_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 1958; CHECK: s_wqm_b64 exec, exec 1959; CHECK: buffer_store_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:4{{$}} 1960; CHECK: s_and_b64 exec, exec, [[LIVE]] 1961; CHECK: buffer_store_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 idxen 1962; CHECK: s_wqm_b64 exec, exec 1963; CHECK: buffer_load_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 offen 1964 1965; CHECK: s_and_b64 exec, exec, [[LIVE]] 1966; CHECK: image_sample 1967; CHECK: buffer_store_dwordx4 1968define amdgpu_ps void @test_alloca(float %data, i32 %a, i32 %idx) nounwind { 1969; GFX9-W64-LABEL: test_alloca: 1970; GFX9-W64: ; %bb.0: ; %entry 1971; GFX9-W64-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 1972; GFX9-W64-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 1973; GFX9-W64-NEXT: s_mov_b32 s10, -1 1974; GFX9-W64-NEXT: s_mov_b32 s11, 0xe00000 1975; GFX9-W64-NEXT: s_add_u32 s8, s8, s0 1976; GFX9-W64-NEXT: s_addc_u32 s9, s9, 0 1977; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec 1978; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1979; GFX9-W64-NEXT: s_and_b64 exec, exec, s[0:1] 1980; GFX9-W64-NEXT: buffer_store_dword v0, off, s[0:3], 0 1981; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1982; GFX9-W64-NEXT: buffer_store_dword v1, off, s[8:11], 0 offset:4 1983; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 1984; GFX9-W64-NEXT: s_and_b64 exec, exec, s[0:1] 1985; GFX9-W64-NEXT: v_mov_b32_e32 v1, 1 1986; GFX9-W64-NEXT: buffer_store_dword v0, v1, s[0:3], 0 idxen 1987; GFX9-W64-NEXT: s_wqm_b64 exec, exec 1988; GFX9-W64-NEXT: v_mov_b32_e32 v0, 4 1989; GFX9-W64-NEXT: v_lshl_add_u32 v0, v2, 2, v0 1990; GFX9-W64-NEXT: buffer_load_dword v0, v0, s[8:11], 0 offen 1991; GFX9-W64-NEXT: s_and_b64 exec, exec, s[0:1] 1992; GFX9-W64-NEXT: s_mov_b32 s0, 0 1993; GFX9-W64-NEXT: s_mov_b32 s1, s0 1994; GFX9-W64-NEXT: s_mov_b32 s2, s0 1995; GFX9-W64-NEXT: s_mov_b32 s3, s0 1996; GFX9-W64-NEXT: s_mov_b32 s4, s0 1997; GFX9-W64-NEXT: s_mov_b32 s5, s0 1998; GFX9-W64-NEXT: s_mov_b32 s6, s0 1999; GFX9-W64-NEXT: s_mov_b32 s7, s0 2000; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2001; GFX9-W64-NEXT: image_sample v[0:3], v0, s[0:7], s[0:3] dmask:0xf 2002; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2003; GFX9-W64-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 2004; GFX9-W64-NEXT: s_endpgm 2005; 2006; GFX10-W32-LABEL: test_alloca: 2007; GFX10-W32: ; %bb.0: ; %entry 2008; GFX10-W32-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 2009; GFX10-W32-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 2010; GFX10-W32-NEXT: s_mov_b32 s10, -1 2011; GFX10-W32-NEXT: s_mov_b32 s11, 0x31c16000 2012; GFX10-W32-NEXT: s_add_u32 s8, s8, s0 2013; GFX10-W32-NEXT: s_addc_u32 s9, s9, 0 2014; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo 2015; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 2016; GFX10-W32-NEXT: v_mov_b32_e32 v3, 1 2017; GFX10-W32-NEXT: v_lshl_add_u32 v2, v2, 2, 4 2018; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s0 2019; GFX10-W32-NEXT: buffer_store_dword v0, off, s[0:3], 0 2020; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 2021; GFX10-W32-NEXT: buffer_store_dword v1, off, s[8:11], 0 offset:4 2022; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 2023; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s0 2024; GFX10-W32-NEXT: buffer_store_dword v0, v3, s[0:3], 0 idxen 2025; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 2026; GFX10-W32-NEXT: buffer_load_dword v0, v2, s[8:11], 0 offen 2027; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s0 2028; GFX10-W32-NEXT: s_mov_b32 s0, 0 2029; GFX10-W32-NEXT: s_mov_b32 s1, s0 2030; GFX10-W32-NEXT: s_mov_b32 s2, s0 2031; GFX10-W32-NEXT: s_mov_b32 s3, s0 2032; GFX10-W32-NEXT: s_mov_b32 s4, s0 2033; GFX10-W32-NEXT: s_mov_b32 s5, s0 2034; GFX10-W32-NEXT: s_mov_b32 s6, s0 2035; GFX10-W32-NEXT: s_mov_b32 s7, s0 2036; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2037; GFX10-W32-NEXT: image_sample v[0:3], v0, s[0:7], s[0:3] dmask:0xf dim:SQ_RSRC_IMG_1D 2038; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2039; GFX10-W32-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 2040; GFX10-W32-NEXT: s_endpgm 2041entry: 2042 %array = alloca [32 x i32], align 4, addrspace(5) 2043 2044 call void @llvm.amdgcn.raw.buffer.store.f32(float %data, <4 x i32> undef, i32 0, i32 0, i32 0) 2045 2046 %s.gep = getelementptr [32 x i32], [32 x i32] addrspace(5)* %array, i32 0, i32 0 2047 store volatile i32 %a, i32 addrspace(5)* %s.gep, align 4 2048 2049 call void @llvm.amdgcn.struct.buffer.store.f32(float %data, <4 x i32> undef, i32 1, i32 0, i32 0, i32 0) 2050 2051 %c.gep = getelementptr [32 x i32], [32 x i32] addrspace(5)* %array, i32 0, i32 %idx 2052 %c = load i32, i32 addrspace(5)* %c.gep, align 4 2053 %c.bc = bitcast i32 %c to float 2054 %t = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0) #0 2055 call void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float> %t, <4 x i32> undef, i32 0, i32 0, i32 0) 2056 2057 ret void 2058} 2059 2060; Must return to exact at the end of a non-void returning shader, 2061; otherwise the EXEC mask exported by the epilog will be wrong. This is true 2062; even if the shader has no kills, because a kill could have happened in a 2063; previous shader fragment. 2064; CHECK-LABEL: {{^}}test_nonvoid_return: 2065; CHECK: s_mov_b64 [[LIVE:s\[[0-9]+:[0-9]+\]]], exec 2066; CHECK: s_wqm_b64 exec, exec 2067; CHECK: s_and_b64 exec, exec, [[LIVE]] 2068; CHECK-NOT: exec 2069define amdgpu_ps <4 x float> @test_nonvoid_return() nounwind { 2070; GFX9-W64-LABEL: test_nonvoid_return: 2071; GFX9-W64: ; %bb.0: 2072; GFX9-W64-NEXT: s_mov_b32 s0, 0 2073; GFX9-W64-NEXT: s_mov_b64 s[8:9], exec 2074; GFX9-W64-NEXT: s_mov_b32 s1, s0 2075; GFX9-W64-NEXT: s_mov_b32 s2, s0 2076; GFX9-W64-NEXT: s_mov_b32 s3, s0 2077; GFX9-W64-NEXT: s_mov_b32 s4, s0 2078; GFX9-W64-NEXT: s_mov_b32 s5, s0 2079; GFX9-W64-NEXT: s_mov_b32 s6, s0 2080; GFX9-W64-NEXT: s_mov_b32 s7, s0 2081; GFX9-W64-NEXT: s_wqm_b64 exec, exec 2082; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[0:3] dmask:0x1 2083; GFX9-W64-NEXT: s_and_b64 exec, exec, s[8:9] 2084; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2085; GFX9-W64-NEXT: image_sample v[0:3], v0, s[0:7], s[0:3] dmask:0xf 2086; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2087; GFX9-W64-NEXT: ; return to shader part epilog 2088; 2089; GFX10-W32-LABEL: test_nonvoid_return: 2090; GFX10-W32: ; %bb.0: 2091; GFX10-W32-NEXT: s_mov_b32 s0, 0 2092; GFX10-W32-NEXT: s_mov_b32 s8, exec_lo 2093; GFX10-W32-NEXT: s_mov_b32 s1, s0 2094; GFX10-W32-NEXT: s_mov_b32 s2, s0 2095; GFX10-W32-NEXT: s_mov_b32 s3, s0 2096; GFX10-W32-NEXT: s_mov_b32 s4, s0 2097; GFX10-W32-NEXT: s_mov_b32 s5, s0 2098; GFX10-W32-NEXT: s_mov_b32 s6, s0 2099; GFX10-W32-NEXT: s_mov_b32 s7, s0 2100; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 2101; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[0:3] dmask:0x1 dim:SQ_RSRC_IMG_1D 2102; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s8 2103; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2104; GFX10-W32-NEXT: image_sample v[0:3], v0, s[0:7], s[0:3] dmask:0xf dim:SQ_RSRC_IMG_1D 2105; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2106; GFX10-W32-NEXT: ; return to shader part epilog 2107 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float undef, <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0) #0 2108 %tex0 = extractelement <4 x float> %tex, i32 0 2109 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0) #0 2110 ret <4 x float> %dtex 2111} 2112 2113; CHECK-LABEL: {{^}}test_nonvoid_return_unreachable: 2114; CHECK: s_mov_b64 [[LIVE:s\[[0-9]+:[0-9]+\]]], exec 2115; CHECK: s_wqm_b64 exec, exec 2116; CHECK: s_and_b64 exec, exec, [[LIVE]] 2117; CHECK-NOT: exec 2118define amdgpu_ps <4 x float> @test_nonvoid_return_unreachable(i32 inreg %c) nounwind { 2119; GFX9-W64-LABEL: test_nonvoid_return_unreachable: 2120; GFX9-W64: ; %bb.0: ; %entry 2121; GFX9-W64-NEXT: s_mov_b32 s4, 0 2122; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec 2123; GFX9-W64-NEXT: s_mov_b32 s5, s4 2124; GFX9-W64-NEXT: s_mov_b32 s6, s4 2125; GFX9-W64-NEXT: s_mov_b32 s7, s4 2126; GFX9-W64-NEXT: s_mov_b32 s8, s4 2127; GFX9-W64-NEXT: s_mov_b32 s9, s4 2128; GFX9-W64-NEXT: s_mov_b32 s10, s4 2129; GFX9-W64-NEXT: s_mov_b32 s11, s4 2130; GFX9-W64-NEXT: s_wqm_b64 exec, exec 2131; GFX9-W64-NEXT: image_sample v0, v0, s[4:11], s[0:3] dmask:0x1 2132; GFX9-W64-NEXT: s_and_b64 exec, exec, s[2:3] 2133; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2134; GFX9-W64-NEXT: image_sample v[0:3], v0, s[4:11], s[0:3] dmask:0xf 2135; GFX9-W64-NEXT: s_cmp_lt_i32 s0, 1 2136; GFX9-W64-NEXT: s_cbranch_scc0 .LBB34_2 2137; GFX9-W64-NEXT: ; %bb.1: ; %else 2138; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2139; GFX9-W64-NEXT: s_branch .LBB34_3 2140; GFX9-W64-NEXT: .LBB34_2: ; %if 2141; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2142; GFX9-W64-NEXT: global_store_dwordx4 v[0:1], v[0:3], off 2143; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2144; GFX9-W64-NEXT: .LBB34_3: 2145; 2146; GFX10-W32-LABEL: test_nonvoid_return_unreachable: 2147; GFX10-W32: ; %bb.0: ; %entry 2148; GFX10-W32-NEXT: s_mov_b32 s4, 0 2149; GFX10-W32-NEXT: s_mov_b32 s1, exec_lo 2150; GFX10-W32-NEXT: s_mov_b32 s5, s4 2151; GFX10-W32-NEXT: s_mov_b32 s6, s4 2152; GFX10-W32-NEXT: s_mov_b32 s7, s4 2153; GFX10-W32-NEXT: s_mov_b32 s8, s4 2154; GFX10-W32-NEXT: s_mov_b32 s9, s4 2155; GFX10-W32-NEXT: s_mov_b32 s10, s4 2156; GFX10-W32-NEXT: s_mov_b32 s11, s4 2157; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 2158; GFX10-W32-NEXT: image_sample v0, v0, s[4:11], s[0:3] dmask:0x1 dim:SQ_RSRC_IMG_1D 2159; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s1 2160; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2161; GFX10-W32-NEXT: image_sample v[0:3], v0, s[4:11], s[0:3] dmask:0xf dim:SQ_RSRC_IMG_1D 2162; GFX10-W32-NEXT: s_cmp_lt_i32 s0, 1 2163; GFX10-W32-NEXT: s_cbranch_scc0 .LBB34_2 2164; GFX10-W32-NEXT: ; %bb.1: ; %else 2165; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2166; GFX10-W32-NEXT: s_branch .LBB34_3 2167; GFX10-W32-NEXT: .LBB34_2: ; %if 2168; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2169; GFX10-W32-NEXT: global_store_dwordx4 v[0:1], v[0:3], off 2170; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 2171; GFX10-W32-NEXT: .LBB34_3: 2172entry: 2173 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float undef, <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0) #0 2174 %tex0 = extractelement <4 x float> %tex, i32 0 2175 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0) #0 2176 %cc = icmp sgt i32 %c, 0 2177 br i1 %cc, label %if, label %else 2178 2179if: 2180 store volatile <4 x float> %dtex, <4 x float> addrspace(1)* undef 2181 unreachable 2182 2183else: 2184 ret <4 x float> %dtex 2185} 2186 2187; Test awareness that s_wqm_b64 clobbers SCC. 2188; CHECK-LABEL: {{^}}test_scc: 2189; CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 2190; CHECK: s_wqm_b64 exec, exec 2191; CHECK: s_cmp_ 2192; CHECK-NEXT: s_cbranch_scc 2193; CHECK: ; %else 2194; CHECK: image_sample 2195; CHECK: ; %if 2196; CHECK: image_sample 2197; CHECK: ; %end 2198; CHECK: s_and_b64 exec, exec, [[ORIG]] 2199define amdgpu_ps <4 x float> @test_scc(i32 inreg %sel, i32 %idx) #1 { 2200; GFX9-W64-LABEL: test_scc: 2201; GFX9-W64: ; %bb.0: ; %main_body 2202; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec 2203; GFX9-W64-NEXT: v_mov_b32_e32 v4, v0 2204; GFX9-W64-NEXT: s_wqm_b64 exec, exec 2205; GFX9-W64-NEXT: s_cmp_lt_i32 s0, 1 2206; GFX9-W64-NEXT: s_cbranch_scc0 .LBB35_2 2207; GFX9-W64-NEXT: ; %bb.1: ; %else 2208; GFX9-W64-NEXT: s_mov_b32 s4, 0 2209; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 2210; GFX9-W64-NEXT: v_mov_b32_e32 v1, 1 2211; GFX9-W64-NEXT: s_mov_b32 s5, s4 2212; GFX9-W64-NEXT: s_mov_b32 s6, s4 2213; GFX9-W64-NEXT: s_mov_b32 s7, s4 2214; GFX9-W64-NEXT: s_mov_b32 s8, s4 2215; GFX9-W64-NEXT: s_mov_b32 s9, s4 2216; GFX9-W64-NEXT: s_mov_b32 s10, s4 2217; GFX9-W64-NEXT: s_mov_b32 s11, s4 2218; GFX9-W64-NEXT: image_sample v[0:3], v[0:1], s[4:11], s[0:3] dmask:0xf 2219; GFX9-W64-NEXT: s_cbranch_execz .LBB35_3 2220; GFX9-W64-NEXT: s_branch .LBB35_4 2221; GFX9-W64-NEXT: .LBB35_2: 2222; GFX9-W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3 2223; GFX9-W64-NEXT: .LBB35_3: ; %if 2224; GFX9-W64-NEXT: s_mov_b32 s4, 0 2225; GFX9-W64-NEXT: s_mov_b32 s5, s4 2226; GFX9-W64-NEXT: s_mov_b32 s6, s4 2227; GFX9-W64-NEXT: s_mov_b32 s7, s4 2228; GFX9-W64-NEXT: s_mov_b32 s8, s4 2229; GFX9-W64-NEXT: s_mov_b32 s9, s4 2230; GFX9-W64-NEXT: s_mov_b32 s10, s4 2231; GFX9-W64-NEXT: s_mov_b32 s11, s4 2232; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2233; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 2234; GFX9-W64-NEXT: image_sample v[0:3], v0, s[4:11], s[0:3] dmask:0xf 2235; GFX9-W64-NEXT: .LBB35_4: ; %end 2236; GFX9-W64-NEXT: s_and_b64 exec, exec, s[2:3] 2237; GFX9-W64-NEXT: v_mov_b32_e32 v5, 1.0 2238; GFX9-W64-NEXT: buffer_store_dword v5, v4, s[0:3], 0 idxen 2239; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2240; GFX9-W64-NEXT: ; return to shader part epilog 2241; 2242; GFX10-W32-LABEL: test_scc: 2243; GFX10-W32: ; %bb.0: ; %main_body 2244; GFX10-W32-NEXT: v_mov_b32_e32 v4, v0 2245; GFX10-W32-NEXT: s_mov_b32 s8, exec_lo 2246; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 2247; GFX10-W32-NEXT: s_cmp_lt_i32 s0, 1 2248; GFX10-W32-NEXT: s_cbranch_scc0 .LBB35_2 2249; GFX10-W32-NEXT: ; %bb.1: ; %else 2250; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 2251; GFX10-W32-NEXT: v_mov_b32_e32 v1, 1 2252; GFX10-W32-NEXT: s_mov_b32 s0, 0 2253; GFX10-W32-NEXT: s_mov_b32 s1, s0 2254; GFX10-W32-NEXT: s_mov_b32 s2, s0 2255; GFX10-W32-NEXT: s_mov_b32 s3, s0 2256; GFX10-W32-NEXT: s_mov_b32 s4, s0 2257; GFX10-W32-NEXT: s_mov_b32 s5, s0 2258; GFX10-W32-NEXT: s_mov_b32 s6, s0 2259; GFX10-W32-NEXT: s_mov_b32 s7, s0 2260; GFX10-W32-NEXT: image_sample v[0:3], v[0:1], s[0:7], s[0:3] dmask:0xf dim:SQ_RSRC_IMG_2D 2261; GFX10-W32-NEXT: s_cbranch_execz .LBB35_3 2262; GFX10-W32-NEXT: s_branch .LBB35_4 2263; GFX10-W32-NEXT: .LBB35_2: 2264; GFX10-W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3 2265; GFX10-W32-NEXT: .LBB35_3: ; %if 2266; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2267; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 2268; GFX10-W32-NEXT: s_mov_b32 s0, 0 2269; GFX10-W32-NEXT: s_mov_b32 s1, s0 2270; GFX10-W32-NEXT: s_mov_b32 s2, s0 2271; GFX10-W32-NEXT: s_mov_b32 s3, s0 2272; GFX10-W32-NEXT: s_mov_b32 s4, s0 2273; GFX10-W32-NEXT: s_mov_b32 s5, s0 2274; GFX10-W32-NEXT: s_mov_b32 s6, s0 2275; GFX10-W32-NEXT: s_mov_b32 s7, s0 2276; GFX10-W32-NEXT: image_sample v[0:3], v0, s[0:7], s[0:3] dmask:0xf dim:SQ_RSRC_IMG_1D 2277; GFX10-W32-NEXT: .LBB35_4: ; %end 2278; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s8 2279; GFX10-W32-NEXT: v_mov_b32_e32 v5, 1.0 2280; GFX10-W32-NEXT: buffer_store_dword v5, v4, s[0:3], 0 idxen 2281; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2282; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 2283; GFX10-W32-NEXT: ; return to shader part epilog 2284main_body: 2285 %cc = icmp sgt i32 %sel, 0 2286 br i1 %cc, label %if, label %else 2287 2288if: 2289 %r.if = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float 0.0, <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0) #0 2290 br label %end 2291 2292else: 2293 %r.else = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32 15, float 0.0, float bitcast (i32 1 to float), <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0) #0 2294 br label %end 2295 2296end: 2297 %r = phi <4 x float> [ %r.if, %if ], [ %r.else, %else ] 2298 call void @llvm.amdgcn.struct.buffer.store.f32(float 1.0, <4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 2299 ret <4 x float> %r 2300} 2301 2302; Check a case of a block being entirely WQM except for a bit of WWM. 2303; There was a bug where it forgot to enter and leave WWM. 2304define amdgpu_ps float @test_wwm_within_wqm(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data) { 2305; GFX9-W64-LABEL: test_wwm_within_wqm: 2306; GFX9-W64: ; %bb.0: ; %main_body 2307; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec 2308; GFX9-W64-NEXT: s_wqm_b64 exec, exec 2309; GFX9-W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 2310; GFX9-W64-NEXT: v_mov_b32_e32 v1, 0 2311; GFX9-W64-NEXT: s_and_saveexec_b64 s[14:15], vcc 2312; GFX9-W64-NEXT: s_cbranch_execz .LBB36_2 2313; GFX9-W64-NEXT: ; %bb.1: ; %IF 2314; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 2315; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2316; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 2317; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2318; GFX9-W64-NEXT: v_cvt_i32_f32_e32 v0, v0 2319; GFX9-W64-NEXT: v_mov_b32_e32 v2, v0 2320; GFX9-W64-NEXT: s_not_b64 exec, exec 2321; GFX9-W64-NEXT: v_mov_b32_e32 v2, 0 2322; GFX9-W64-NEXT: s_not_b64 exec, exec 2323; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], -1 2324; GFX9-W64-NEXT: ds_swizzle_b32 v2, v2 offset:swizzle(SWAP,2) 2325; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1] 2326; GFX9-W64-NEXT: s_waitcnt lgkmcnt(0) 2327; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2 2328; GFX9-W64-NEXT: v_cvt_f32_i32_e32 v1, v0 2329; GFX9-W64-NEXT: .LBB36_2: ; %ENDIF 2330; GFX9-W64-NEXT: s_or_b64 exec, exec, s[14:15] 2331; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 2332; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 2333; GFX9-W64-NEXT: ; return to shader part epilog 2334; 2335; GFX10-W32-LABEL: test_wwm_within_wqm: 2336; GFX10-W32: ; %bb.0: ; %main_body 2337; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo 2338; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 2339; GFX10-W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1 2340; GFX10-W32-NEXT: v_mov_b32_e32 v1, 0 2341; GFX10-W32-NEXT: s_and_saveexec_b32 s13, vcc_lo 2342; GFX10-W32-NEXT: s_cbranch_execz .LBB36_2 2343; GFX10-W32-NEXT: ; %bb.1: ; %IF 2344; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 2345; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2346; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 2347; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2348; GFX10-W32-NEXT: v_cvt_i32_f32_e32 v0, v0 2349; GFX10-W32-NEXT: v_mov_b32_e32 v2, v0 2350; GFX10-W32-NEXT: s_not_b32 exec_lo, exec_lo 2351; GFX10-W32-NEXT: v_mov_b32_e32 v2, 0 2352; GFX10-W32-NEXT: s_not_b32 exec_lo, exec_lo 2353; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 2354; GFX10-W32-NEXT: ds_swizzle_b32 v2, v2 offset:swizzle(SWAP,2) 2355; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 2356; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0) 2357; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2 2358; GFX10-W32-NEXT: v_cvt_f32_i32_e32 v1, v0 2359; GFX10-W32-NEXT: .LBB36_2: ; %ENDIF 2360; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s13 2361; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 2362; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 2363; GFX10-W32-NEXT: ; return to shader part epilog 2364main_body: 2365 %c.bc = bitcast i32 %c to float 2366 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 2367 %tex0 = extractelement <4 x float> %tex, i32 0 2368 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 2369 %cmp = icmp eq i32 %z, 0 2370 br i1 %cmp, label %IF, label %ENDIF 2371 2372IF: 2373 %dataf = extractelement <4 x float> %dtex, i32 0 2374 %data1 = fptosi float %dataf to i32 2375 %data2 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %data1, i32 0) 2376 %data3 = call i32 @llvm.amdgcn.ds.swizzle(i32 %data2, i32 2079) 2377 %data4 = call i32 @llvm.amdgcn.wwm.i32(i32 %data3) 2378 %data4f = sitofp i32 %data4 to float 2379 br label %ENDIF 2380 2381ENDIF: 2382 %r = phi float [ %data4f, %IF ], [ 0.0, %main_body ] 2383 ret float %r 2384} 2385 2386; Check that WWM is triggered by the strict_wwm intrinsic. 2387define amdgpu_ps float @test_strict_wwm1(i32 inreg %idx0, i32 inreg %idx1) { 2388; GFX9-W64-LABEL: test_strict_wwm1: 2389; GFX9-W64: ; %bb.0: ; %main_body 2390; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1 2391; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 2392; GFX9-W64-NEXT: v_mov_b32_e32 v2, s1 2393; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 2394; GFX9-W64-NEXT: s_nop 0 2395; GFX9-W64-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen 2396; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2397; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v2 2398; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 2399; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 2400; GFX9-W64-NEXT: ; return to shader part epilog 2401; 2402; GFX10-W32-LABEL: test_strict_wwm1: 2403; GFX10-W32: ; %bb.0: ; %main_body 2404; GFX10-W32-NEXT: s_or_saveexec_b32 s2, -1 2405; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 2406; GFX10-W32-NEXT: v_mov_b32_e32 v2, s1 2407; GFX10-W32-NEXT: s_clause 0x1 2408; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 2409; GFX10-W32-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen 2410; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2411; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v2 2412; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2 2413; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 2414; GFX10-W32-NEXT: ; return to shader part epilog 2415main_body: 2416 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 2417 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 2418 %out = fadd float %src0, %src1 2419 %out.0 = call float @llvm.amdgcn.strict.wwm.f32(float %out) 2420 ret float %out.0 2421} 2422 2423; Same as above, but with an integer type. 2424define amdgpu_ps float @test_strict_wwm2(i32 inreg %idx0, i32 inreg %idx1) { 2425; GFX9-W64-LABEL: test_strict_wwm2: 2426; GFX9-W64: ; %bb.0: ; %main_body 2427; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1 2428; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 2429; GFX9-W64-NEXT: v_mov_b32_e32 v2, s1 2430; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 2431; GFX9-W64-NEXT: s_nop 0 2432; GFX9-W64-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen 2433; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2434; GFX9-W64-NEXT: v_add_u32_e32 v1, v1, v2 2435; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 2436; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 2437; GFX9-W64-NEXT: ; return to shader part epilog 2438; 2439; GFX10-W32-LABEL: test_strict_wwm2: 2440; GFX10-W32: ; %bb.0: ; %main_body 2441; GFX10-W32-NEXT: s_or_saveexec_b32 s2, -1 2442; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 2443; GFX10-W32-NEXT: v_mov_b32_e32 v2, s1 2444; GFX10-W32-NEXT: s_clause 0x1 2445; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 2446; GFX10-W32-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen 2447; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2448; GFX10-W32-NEXT: v_add_nc_u32_e32 v1, v1, v2 2449; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2 2450; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 2451; GFX10-W32-NEXT: ; return to shader part epilog 2452main_body: 2453 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 2454 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 2455 %src0.0 = bitcast float %src0 to i32 2456 %src1.0 = bitcast float %src1 to i32 2457 %out = add i32 %src0.0, %src1.0 2458 %out.0 = call i32 @llvm.amdgcn.strict.wwm.i32(i32 %out) 2459 %out.1 = bitcast i32 %out.0 to float 2460 ret float %out.1 2461} 2462 2463; Check that we don't leave WWM on for computations that don't require WWM, 2464; since that will lead clobbering things that aren't supposed to be clobbered 2465; in cases like this. 2466; We enforce this by checking that v_add gets emitted in the same block as 2467; WWM computations. 2468define amdgpu_ps float @test_strict_wwm3(i32 inreg %idx) { 2469; GFX9-W64-LABEL: test_strict_wwm3: 2470; GFX9-W64: ; %bb.0: ; %main_body 2471; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 2472; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 2473; GFX9-W64-NEXT: v_cmp_gt_u32_e32 vcc, 16, v0 2474; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 2475; GFX9-W64-NEXT: s_and_saveexec_b64 s[2:3], vcc 2476; GFX9-W64-NEXT: s_cbranch_execz .LBB39_2 2477; GFX9-W64-NEXT: ; %bb.1: ; %if 2478; GFX9-W64-NEXT: s_or_saveexec_b64 s[4:5], -1 2479; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 2480; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 2481; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2482; GFX9-W64-NEXT: v_add_f32_e32 v2, v1, v1 2483; GFX9-W64-NEXT: s_mov_b64 exec, s[4:5] 2484; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2 2485; GFX9-W64-NEXT: v_add_f32_e32 v0, v1, v0 2486; GFX9-W64-NEXT: .LBB39_2: ; %endif 2487; GFX9-W64-NEXT: s_or_b64 exec, exec, s[2:3] 2488; GFX9-W64-NEXT: ; return to shader part epilog 2489; 2490; GFX10-W32-LABEL: test_strict_wwm3: 2491; GFX10-W32: ; %bb.0: ; %main_body 2492; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 2493; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 2494; GFX10-W32-NEXT: v_cmp_gt_u32_e32 vcc_lo, 16, v0 2495; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 2496; GFX10-W32-NEXT: s_and_saveexec_b32 s1, vcc_lo 2497; GFX10-W32-NEXT: s_cbranch_execz .LBB39_2 2498; GFX10-W32-NEXT: ; %bb.1: ; %if 2499; GFX10-W32-NEXT: s_or_saveexec_b32 s2, -1 2500; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 2501; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 2502; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2503; GFX10-W32-NEXT: v_add_f32_e32 v2, v1, v1 2504; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2 2505; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2 2506; GFX10-W32-NEXT: v_add_f32_e32 v0, v1, v0 2507; GFX10-W32-NEXT: .LBB39_2: ; %endif 2508; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s1 2509; GFX10-W32-NEXT: ; return to shader part epilog 2510main_body: 2511 ; use mbcnt to make sure the branch is divergent 2512 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 2513 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 2514 %cc = icmp uge i32 %hi, 16 2515 br i1 %cc, label %endif, label %if 2516 2517if: 2518 %src = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 2519 %out = fadd float %src, %src 2520 %out.0 = call float @llvm.amdgcn.strict.wwm.f32(float %out) 2521 %out.1 = fadd float %src, %out.0 2522 br label %endif 2523 2524endif: 2525 %out.2 = phi float [ %out.1, %if ], [ 0.0, %main_body ] 2526 ret float %out.2 2527} 2528 2529; Check that WWM writes aren't coalesced with non-WWM writes, since the WWM 2530; write could clobber disabled channels in the non-WWM one. 2531; We enforce this by checking that v_mov gets emitted in the same block as 2532; WWM computations. 2533define amdgpu_ps float @test_strict_wwm4(i32 inreg %idx) { 2534; GFX9-W64-LABEL: test_strict_wwm4: 2535; GFX9-W64: ; %bb.0: ; %main_body 2536; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 2537; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 2538; GFX9-W64-NEXT: v_cmp_gt_u32_e32 vcc, 16, v0 2539; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 2540; GFX9-W64-NEXT: s_and_saveexec_b64 s[2:3], vcc 2541; GFX9-W64-NEXT: s_cbranch_execz .LBB40_2 2542; GFX9-W64-NEXT: ; %bb.1: ; %if 2543; GFX9-W64-NEXT: s_or_saveexec_b64 s[4:5], -1 2544; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 2545; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 2546; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2547; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v1 2548; GFX9-W64-NEXT: s_mov_b64 exec, s[4:5] 2549; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 2550; GFX9-W64-NEXT: .LBB40_2: ; %endif 2551; GFX9-W64-NEXT: s_or_b64 exec, exec, s[2:3] 2552; GFX9-W64-NEXT: ; return to shader part epilog 2553; 2554; GFX10-W32-LABEL: test_strict_wwm4: 2555; GFX10-W32: ; %bb.0: ; %main_body 2556; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 2557; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 2558; GFX10-W32-NEXT: v_cmp_gt_u32_e32 vcc_lo, 16, v0 2559; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 2560; GFX10-W32-NEXT: s_and_saveexec_b32 s1, vcc_lo 2561; GFX10-W32-NEXT: s_cbranch_execz .LBB40_2 2562; GFX10-W32-NEXT: ; %bb.1: ; %if 2563; GFX10-W32-NEXT: s_or_saveexec_b32 s2, -1 2564; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 2565; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 2566; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2567; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v1 2568; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2 2569; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 2570; GFX10-W32-NEXT: .LBB40_2: ; %endif 2571; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s1 2572; GFX10-W32-NEXT: ; return to shader part epilog 2573main_body: 2574 ; use mbcnt to make sure the branch is divergent 2575 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 2576 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 2577 %cc = icmp uge i32 %hi, 16 2578 br i1 %cc, label %endif, label %if 2579 2580if: 2581 %src = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 2582 %out = fadd float %src, %src 2583 %out.0 = call float @llvm.amdgcn.strict.wwm.f32(float %out) 2584 br label %endif 2585 2586endif: 2587 %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ] 2588 ret float %out.1 2589} 2590 2591; Make sure the transition from Exact to WWM then WQM works properly. 2592define amdgpu_ps float @test_strict_wwm5(i32 inreg %idx0, i32 inreg %idx1) { 2593; GFX9-W64-LABEL: test_strict_wwm5: 2594; GFX9-W64: ; %bb.0: ; %main_body 2595; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec 2596; GFX9-W64-NEXT: v_mov_b32_e32 v0, s0 2597; GFX9-W64-NEXT: buffer_load_dword v2, v0, s[0:3], 0 idxen 2598; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2599; GFX9-W64-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen 2600; GFX9-W64-NEXT: s_or_saveexec_b64 s[4:5], -1 2601; GFX9-W64-NEXT: v_mov_b32_e32 v1, s1 2602; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 2603; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2604; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v1 2605; GFX9-W64-NEXT: s_mov_b64 exec, s[4:5] 2606; GFX9-W64-NEXT: s_wqm_b64 exec, exec 2607; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 2608; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v0 2609; GFX9-W64-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec killed $exec 2610; GFX9-W64-NEXT: s_and_b64 exec, exec, s[2:3] 2611; GFX9-W64-NEXT: ; return to shader part epilog 2612; 2613; GFX10-W32-LABEL: test_strict_wwm5: 2614; GFX10-W32: ; %bb.0: ; %main_body 2615; GFX10-W32-NEXT: v_mov_b32_e32 v0, s0 2616; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo 2617; GFX10-W32-NEXT: buffer_load_dword v2, v0, s[0:3], 0 idxen 2618; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 2619; GFX10-W32-NEXT: v_mov_b32_e32 v1, s1 2620; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 2621; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2622; GFX10-W32-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen 2623; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 2624; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen 2625; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2626; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v1 2627; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 2628; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 2629; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 2630; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v0 2631; GFX10-W32-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec killed $exec 2632; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s2 2633; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 2634; GFX10-W32-NEXT: ; return to shader part epilog 2635main_body: 2636 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 2637 call void @llvm.amdgcn.struct.buffer.store.f32(float %src0, <4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 2638 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 2639 %temp = fadd float %src1, %src1 2640 %temp.0 = call float @llvm.amdgcn.strict.wwm.f32(float %temp) 2641 %out = fadd float %temp.0, %temp.0 2642 %out.0 = call float @llvm.amdgcn.wqm.f32(float %out) 2643 ret float %out.0 2644} 2645 2646; Check that WWM is turned on correctly across basic block boundaries. 2647; if..then..endif version 2648;SI-CHECK: buffer_load_dword 2649;VI-CHECK: flat_load_dword 2650;SI-CHECK: buffer_load_dword 2651;VI-CHECK: flat_load_dword 2652define amdgpu_ps float @test_strict_wwm6_then() { 2653; GFX9-W64-LABEL: test_strict_wwm6_then: 2654; GFX9-W64: ; %bb.0: ; %main_body 2655; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], -1 2656; GFX9-W64-NEXT: global_load_dword v1, v[3:4], off glc 2657; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2658; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1] 2659; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 2660; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 2661; GFX9-W64-NEXT: v_cmp_gt_u32_e32 vcc, 16, v0 2662; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 2663; GFX9-W64-NEXT: s_and_saveexec_b64 s[0:1], vcc 2664; GFX9-W64-NEXT: s_cbranch_execz .LBB42_2 2665; GFX9-W64-NEXT: ; %bb.1: ; %if 2666; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1 2667; GFX9-W64-NEXT: global_load_dword v2, v[3:4], off glc 2668; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2669; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v2 2670; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 2671; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 2672; GFX9-W64-NEXT: .LBB42_2: ; %endif 2673; GFX9-W64-NEXT: s_or_b64 exec, exec, s[0:1] 2674; GFX9-W64-NEXT: ; return to shader part epilog 2675; 2676; GFX10-W32-LABEL: test_strict_wwm6_then: 2677; GFX10-W32: ; %bb.0: ; %main_body 2678; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 2679; GFX10-W32-NEXT: global_load_dword v1, v[3:4], off glc dlc 2680; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2681; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 2682; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 2683; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0 2684; GFX10-W32-NEXT: v_cmp_gt_u32_e32 vcc_lo, 16, v0 2685; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 2686; GFX10-W32-NEXT: s_and_saveexec_b32 s0, vcc_lo 2687; GFX10-W32-NEXT: s_cbranch_execz .LBB42_2 2688; GFX10-W32-NEXT: ; %bb.1: ; %if 2689; GFX10-W32-NEXT: s_or_saveexec_b32 s1, -1 2690; GFX10-W32-NEXT: global_load_dword v2, v[3:4], off glc dlc 2691; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2692; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v2 2693; GFX10-W32-NEXT: s_mov_b32 exec_lo, s1 2694; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 2695; GFX10-W32-NEXT: .LBB42_2: ; %endif 2696; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s0 2697; GFX10-W32-NEXT: ; return to shader part epilog 2698main_body: 2699 %src0 = load volatile float, float addrspace(1)* undef 2700 ; use mbcnt to make sure the branch is divergent 2701 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 2702 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 2703 %cc = icmp uge i32 %hi, 16 2704 br i1 %cc, label %endif, label %if 2705 2706if: 2707 %src1 = load volatile float, float addrspace(1)* undef 2708 %out = fadd float %src0, %src1 2709 %out.0 = call float @llvm.amdgcn.strict.wwm.f32(float %out) 2710 br label %endif 2711 2712endif: 2713 %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ] 2714 ret float %out.1 2715} 2716 2717; Check that WWM is turned on correctly across basic block boundaries. 2718; loop version 2719define amdgpu_ps float @test_strict_wwm6_loop() { 2720; GFX9-W64-LABEL: test_strict_wwm6_loop: 2721; GFX9-W64: ; %bb.0: ; %main_body 2722; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], -1 2723; GFX9-W64-NEXT: global_load_dword v1, v[3:4], off glc 2724; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2725; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1] 2726; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 2727; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v0 2728; GFX9-W64-NEXT: s_mov_b64 s[0:1], 0 2729; GFX9-W64-NEXT: .LBB43_1: ; %loop 2730; GFX9-W64-NEXT: ; =>This Inner Loop Header: Depth=1 2731; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1 2732; GFX9-W64-NEXT: global_load_dword v2, v[3:4], off glc 2733; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2734; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 2735; GFX9-W64-NEXT: v_add_u32_e32 v3, -1, v3 2736; GFX9-W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3 2737; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1 2738; GFX9-W64-NEXT: v_add_f32_e32 v2, v1, v2 2739; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 2740; GFX9-W64-NEXT: s_or_b64 s[0:1], vcc, s[0:1] 2741; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2 2742; GFX9-W64-NEXT: s_andn2_b64 exec, exec, s[0:1] 2743; GFX9-W64-NEXT: s_cbranch_execnz .LBB43_1 2744; GFX9-W64-NEXT: ; %bb.2: ; %endloop 2745; GFX9-W64-NEXT: s_or_b64 exec, exec, s[0:1] 2746; GFX9-W64-NEXT: ; return to shader part epilog 2747; 2748; GFX10-W32-LABEL: test_strict_wwm6_loop: 2749; GFX10-W32: ; %bb.0: ; %main_body 2750; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 2751; GFX10-W32-NEXT: global_load_dword v1, v[3:4], off glc dlc 2752; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2753; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 2754; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0 2755; GFX10-W32-NEXT: s_mov_b32 s0, 0 2756; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v0 2757; GFX10-W32-NEXT: .LBB43_1: ; %loop 2758; GFX10-W32-NEXT: ; =>This Inner Loop Header: Depth=1 2759; GFX10-W32-NEXT: s_or_saveexec_b32 s1, -1 2760; GFX10-W32-NEXT: global_load_dword v2, v[3:4], off glc dlc 2761; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2762; GFX10-W32-NEXT: s_mov_b32 exec_lo, s1 2763; GFX10-W32-NEXT: v_add_nc_u32_e32 v3, -1, v3 2764; GFX10-W32-NEXT: s_or_saveexec_b32 s1, -1 2765; GFX10-W32-NEXT: v_add_f32_e32 v2, v1, v2 2766; GFX10-W32-NEXT: s_mov_b32 exec_lo, s1 2767; GFX10-W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v3 2768; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2 2769; GFX10-W32-NEXT: s_or_b32 s0, vcc_lo, s0 2770; GFX10-W32-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 2771; GFX10-W32-NEXT: s_cbranch_execnz .LBB43_1 2772; GFX10-W32-NEXT: ; %bb.2: ; %endloop 2773; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s0 2774; GFX10-W32-NEXT: ; return to shader part epilog 2775main_body: 2776 %src0 = load volatile float, float addrspace(1)* undef 2777 ; use mbcnt to make sure the branch is divergent 2778 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 2779 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 2780 br label %loop 2781 2782loop: 2783 %counter = phi i32 [ %hi, %main_body ], [ %counter.1, %loop ] 2784 %src1 = load volatile float, float addrspace(1)* undef 2785 %out = fadd float %src0, %src1 2786 %out.0 = call float @llvm.amdgcn.strict.wwm.f32(float %out) 2787 %counter.1 = sub i32 %counter, 1 2788 %cc = icmp ne i32 %counter.1, 0 2789 br i1 %cc, label %loop, label %endloop 2790 2791endloop: 2792 ret float %out.0 2793} 2794 2795; Check that @llvm.amdgcn.set.inactive disables WWM. 2796define amdgpu_ps void @test_strict_wwm_set_inactive1(i32 inreg %idx) { 2797; GFX9-W64-LABEL: test_strict_wwm_set_inactive1: 2798; GFX9-W64: ; %bb.0: ; %main_body 2799; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 2800; GFX9-W64-NEXT: buffer_load_dword v2, v1, s[0:3], 0 idxen 2801; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2802; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2 2803; GFX9-W64-NEXT: s_not_b64 exec, exec 2804; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 2805; GFX9-W64-NEXT: s_not_b64 exec, exec 2806; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], -1 2807; GFX9-W64-NEXT: v_add_u32_e32 v0, v0, v0 2808; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1] 2809; GFX9-W64-NEXT: v_mov_b32_e32 v2, v0 2810; GFX9-W64-NEXT: buffer_store_dword v2, v1, s[0:3], 0 idxen 2811; GFX9-W64-NEXT: s_endpgm 2812; 2813; GFX10-W32-LABEL: test_strict_wwm_set_inactive1: 2814; GFX10-W32: ; %bb.0: ; %main_body 2815; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 2816; GFX10-W32-NEXT: buffer_load_dword v2, v1, s[0:3], 0 idxen 2817; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2818; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2 2819; GFX10-W32-NEXT: s_not_b32 exec_lo, exec_lo 2820; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 2821; GFX10-W32-NEXT: s_not_b32 exec_lo, exec_lo 2822; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 2823; GFX10-W32-NEXT: v_add_nc_u32_e32 v0, v0, v0 2824; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 2825; GFX10-W32-NEXT: v_mov_b32_e32 v2, v0 2826; GFX10-W32-NEXT: buffer_store_dword v2, v1, s[0:3], 0 idxen 2827; GFX10-W32-NEXT: s_endpgm 2828main_body: 2829 %src = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 2830 %src.0 = bitcast float %src to i32 2831 %src.1 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %src.0, i32 0) 2832 %out = add i32 %src.1, %src.1 2833 %out.0 = call i32 @llvm.amdgcn.strict.wwm.i32(i32 %out) 2834 %out.1 = bitcast i32 %out.0 to float 2835 call void @llvm.amdgcn.struct.buffer.store.f32(float %out.1, <4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 2836 ret void 2837} 2838 2839; Check a case of a block being entirely WQM except for a bit of WWM. 2840; There was a bug where it forgot to enter and leave WWM. 2841define amdgpu_ps float @test_strict_wwm_within_wqm(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data) { 2842; GFX9-W64-LABEL: test_strict_wwm_within_wqm: 2843; GFX9-W64: ; %bb.0: ; %main_body 2844; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec 2845; GFX9-W64-NEXT: s_wqm_b64 exec, exec 2846; GFX9-W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 2847; GFX9-W64-NEXT: v_mov_b32_e32 v1, 0 2848; GFX9-W64-NEXT: s_and_saveexec_b64 s[14:15], vcc 2849; GFX9-W64-NEXT: s_cbranch_execz .LBB45_2 2850; GFX9-W64-NEXT: ; %bb.1: ; %IF 2851; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 2852; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2853; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 2854; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2855; GFX9-W64-NEXT: v_cvt_i32_f32_e32 v0, v0 2856; GFX9-W64-NEXT: v_mov_b32_e32 v2, v0 2857; GFX9-W64-NEXT: s_not_b64 exec, exec 2858; GFX9-W64-NEXT: v_mov_b32_e32 v2, 0 2859; GFX9-W64-NEXT: s_not_b64 exec, exec 2860; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], -1 2861; GFX9-W64-NEXT: ds_swizzle_b32 v2, v2 offset:swizzle(SWAP,2) 2862; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1] 2863; GFX9-W64-NEXT: s_waitcnt lgkmcnt(0) 2864; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2 2865; GFX9-W64-NEXT: v_cvt_f32_i32_e32 v1, v0 2866; GFX9-W64-NEXT: .LBB45_2: ; %ENDIF 2867; GFX9-W64-NEXT: s_or_b64 exec, exec, s[14:15] 2868; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 2869; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1 2870; GFX9-W64-NEXT: ; return to shader part epilog 2871; 2872; GFX10-W32-LABEL: test_strict_wwm_within_wqm: 2873; GFX10-W32: ; %bb.0: ; %main_body 2874; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo 2875; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 2876; GFX10-W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1 2877; GFX10-W32-NEXT: v_mov_b32_e32 v1, 0 2878; GFX10-W32-NEXT: s_and_saveexec_b32 s13, vcc_lo 2879; GFX10-W32-NEXT: s_cbranch_execz .LBB45_2 2880; GFX10-W32-NEXT: ; %bb.1: ; %IF 2881; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 2882; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2883; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 2884; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2885; GFX10-W32-NEXT: v_cvt_i32_f32_e32 v0, v0 2886; GFX10-W32-NEXT: v_mov_b32_e32 v2, v0 2887; GFX10-W32-NEXT: s_not_b32 exec_lo, exec_lo 2888; GFX10-W32-NEXT: v_mov_b32_e32 v2, 0 2889; GFX10-W32-NEXT: s_not_b32 exec_lo, exec_lo 2890; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 2891; GFX10-W32-NEXT: ds_swizzle_b32 v2, v2 offset:swizzle(SWAP,2) 2892; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 2893; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0) 2894; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2 2895; GFX10-W32-NEXT: v_cvt_f32_i32_e32 v1, v0 2896; GFX10-W32-NEXT: .LBB45_2: ; %ENDIF 2897; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s13 2898; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 2899; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1 2900; GFX10-W32-NEXT: ; return to shader part epilog 2901main_body: 2902 %c.bc = bitcast i32 %c to float 2903 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 2904 %tex0 = extractelement <4 x float> %tex, i32 0 2905 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 2906 %cmp = icmp eq i32 %z, 0 2907 br i1 %cmp, label %IF, label %ENDIF 2908 2909IF: 2910 %dataf = extractelement <4 x float> %dtex, i32 0 2911 %data1 = fptosi float %dataf to i32 2912 %data2 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %data1, i32 0) 2913 %data3 = call i32 @llvm.amdgcn.ds.swizzle(i32 %data2, i32 2079) 2914 %data4 = call i32 @llvm.amdgcn.strict.wwm.i32(i32 %data3) 2915 %data4f = sitofp i32 %data4 to float 2916 br label %ENDIF 2917 2918ENDIF: 2919 %r = phi float [ %data4f, %IF ], [ 0.0, %main_body ] 2920 ret float %r 2921} 2922 2923; Check a case of a block being entirely WQM except for a bit of STRICT WQM. 2924define amdgpu_ps float @test_strict_wqm_within_wqm(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data) { 2925; GFX9-W64-LABEL: test_strict_wqm_within_wqm: 2926; GFX9-W64: ; %bb.0: ; %main_body 2927; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec 2928; GFX9-W64-NEXT: s_wqm_b64 exec, exec 2929; GFX9-W64-NEXT: s_mov_b64 s[14:15], exec 2930; GFX9-W64-NEXT: s_wqm_b64 exec, exec 2931; GFX9-W64-NEXT: v_mov_b32_e32 v2, v0 2932; GFX9-W64-NEXT: s_mov_b64 exec, s[14:15] 2933; GFX9-W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1 2934; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0 2935; GFX9-W64-NEXT: s_and_saveexec_b64 s[14:15], vcc 2936; GFX9-W64-NEXT: s_cbranch_execz .LBB46_2 2937; GFX9-W64-NEXT: ; %bb.1: ; %IF 2938; GFX9-W64-NEXT: s_mov_b64 s[16:17], exec 2939; GFX9-W64-NEXT: s_wqm_b64 exec, exec 2940; GFX9-W64-NEXT: image_sample v2, v2, s[0:7], s[8:11] dmask:0x1 2941; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2942; GFX9-W64-NEXT: image_sample v2, v2, s[0:7], s[8:11] dmask:0x1 2943; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 2944; GFX9-W64-NEXT: v_cvt_i32_f32_e32 v2, v2 2945; GFX9-W64-NEXT: ds_swizzle_b32 v2, v2 offset:swizzle(SWAP,2) 2946; GFX9-W64-NEXT: s_mov_b64 exec, s[16:17] 2947; GFX9-W64-NEXT: s_waitcnt lgkmcnt(0) 2948; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2 2949; GFX9-W64-NEXT: v_cvt_f32_i32_e32 v0, v0 2950; GFX9-W64-NEXT: .LBB46_2: ; %ENDIF 2951; GFX9-W64-NEXT: s_or_b64 exec, exec, s[14:15] 2952; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13] 2953; GFX9-W64-NEXT: ; return to shader part epilog 2954; 2955; GFX10-W32-LABEL: test_strict_wqm_within_wqm: 2956; GFX10-W32: ; %bb.0: ; %main_body 2957; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo 2958; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 2959; GFX10-W32-NEXT: s_mov_b32 s13, exec_lo 2960; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 2961; GFX10-W32-NEXT: v_mov_b32_e32 v2, v0 2962; GFX10-W32-NEXT: s_mov_b32 exec_lo, s13 2963; GFX10-W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1 2964; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0 2965; GFX10-W32-NEXT: s_and_saveexec_b32 s13, vcc_lo 2966; GFX10-W32-NEXT: s_cbranch_execz .LBB46_2 2967; GFX10-W32-NEXT: ; %bb.1: ; %IF 2968; GFX10-W32-NEXT: s_mov_b32 s14, exec_lo 2969; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 2970; GFX10-W32-NEXT: image_sample v2, v2, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 2971; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2972; GFX10-W32-NEXT: image_sample v2, v2, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D 2973; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 2974; GFX10-W32-NEXT: v_cvt_i32_f32_e32 v2, v2 2975; GFX10-W32-NEXT: ds_swizzle_b32 v2, v2 offset:swizzle(SWAP,2) 2976; GFX10-W32-NEXT: s_mov_b32 exec_lo, s14 2977; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0) 2978; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2 2979; GFX10-W32-NEXT: v_cvt_f32_i32_e32 v0, v0 2980; GFX10-W32-NEXT: .LBB46_2: ; %ENDIF 2981; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s13 2982; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12 2983; GFX10-W32-NEXT: ; return to shader part epilog 2984main_body: 2985 %c.bc = bitcast i32 %c to float 2986 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 2987 %tex0 = extractelement <4 x float> %tex, i32 0 2988 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 2989 %cmp = icmp eq i32 %z, 0 2990 br i1 %cmp, label %IF, label %ENDIF 2991 2992IF: 2993 %dataf = extractelement <4 x float> %dtex, i32 0 2994 %data1 = fptosi float %dataf to i32 2995 %data2 = call i32 @llvm.amdgcn.ds.swizzle(i32 %data1, i32 2079) 2996 %data3 = call i32 @llvm.amdgcn.strict.wqm.i32(i32 %data2) 2997 %data3f = sitofp i32 %data3 to float 2998 br label %ENDIF 2999 3000ENDIF: 3001 %r = phi float [ %data3f, %IF ], [ 0.0, %main_body ] 3002 ret float %r 3003} 3004 3005;TODO: StrictWQM -> WQM transition could be improved. WQM could use the exec from the previous state instead of calling s_wqm again. 3006define amdgpu_ps float @test_strict_wqm_strict_wwm_wqm(i32 inreg %idx0, i32 inreg %idx1, <4 x i32> inreg %res, <4 x i32> inreg %res2, float %inp, <8 x i32> inreg %res3) { 3007; GFX9-W64-LABEL: test_strict_wqm_strict_wwm_wqm: 3008; GFX9-W64: ; %bb.0: ; %main_body 3009; GFX9-W64-NEXT: s_mov_b64 s[28:29], exec 3010; GFX9-W64-NEXT: s_mov_b32 s19, s17 3011; GFX9-W64-NEXT: s_mov_b64 s[30:31], exec 3012; GFX9-W64-NEXT: s_wqm_b64 exec, exec 3013; GFX9-W64-NEXT: s_mov_b32 s23, s5 3014; GFX9-W64-NEXT: s_mov_b32 s22, s4 3015; GFX9-W64-NEXT: s_mov_b32 s21, s3 3016; GFX9-W64-NEXT: s_mov_b32 s20, s2 3017; GFX9-W64-NEXT: s_mov_b32 s27, s9 3018; GFX9-W64-NEXT: s_mov_b32 s26, s8 3019; GFX9-W64-NEXT: s_mov_b32 s25, s7 3020; GFX9-W64-NEXT: s_mov_b32 s24, s6 3021; GFX9-W64-NEXT: s_mov_b32 s18, s16 3022; GFX9-W64-NEXT: s_mov_b32 s17, s15 3023; GFX9-W64-NEXT: s_mov_b32 s16, s14 3024; GFX9-W64-NEXT: s_mov_b32 s15, s13 3025; GFX9-W64-NEXT: s_mov_b32 s14, s12 3026; GFX9-W64-NEXT: s_mov_b32 s13, s11 3027; GFX9-W64-NEXT: s_mov_b32 s12, s10 3028; GFX9-W64-NEXT: v_mov_b32_e32 v1, s1 3029; GFX9-W64-NEXT: s_mov_b64 exec, s[30:31] 3030; GFX9-W64-NEXT: buffer_store_dword v0, v1, s[20:23], 0 idxen 3031; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec 3032; GFX9-W64-NEXT: s_wqm_b64 exec, exec 3033; GFX9-W64-NEXT: buffer_load_dword v2, v1, s[20:23], 0 idxen 3034; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 3035; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1 3036; GFX9-W64-NEXT: v_mov_b32_e32 v3, s0 3037; GFX9-W64-NEXT: buffer_load_dword v3, v3, s[24:27], 0 idxen 3038; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 3039; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec 3040; GFX9-W64-NEXT: s_wqm_b64 exec, exec 3041; GFX9-W64-NEXT: s_waitcnt vmcnt(1) 3042; GFX9-W64-NEXT: v_add_f32_e32 v2, v2, v2 3043; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1] 3044; GFX9-W64-NEXT: s_wqm_b64 exec, exec 3045; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2 3046; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v0 3047; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 3048; GFX9-W64-NEXT: v_mov_b32_e32 v4, v3 3049; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v4 3050; GFX9-W64-NEXT: s_and_b64 exec, exec, s[28:29] 3051; GFX9-W64-NEXT: image_sample v0, v0, s[12:19], s[20:23] dmask:0x1 3052; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 3053; GFX9-W64-NEXT: buffer_store_dword v0, v1, s[20:23], 0 idxen 3054; GFX9-W64-NEXT: buffer_load_dword v0, v1, s[20:23], 0 idxen 3055; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 3056; GFX9-W64-NEXT: ; return to shader part epilog 3057; 3058; GFX10-W32-LABEL: test_strict_wqm_strict_wwm_wqm: 3059; GFX10-W32: ; %bb.0: ; %main_body 3060; GFX10-W32-NEXT: s_mov_b32 s28, exec_lo 3061; GFX10-W32-NEXT: s_mov_b32 s19, s17 3062; GFX10-W32-NEXT: s_mov_b32 s29, exec_lo 3063; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 3064; GFX10-W32-NEXT: s_mov_b32 s23, s5 3065; GFX10-W32-NEXT: s_mov_b32 s22, s4 3066; GFX10-W32-NEXT: s_mov_b32 s21, s3 3067; GFX10-W32-NEXT: s_mov_b32 s20, s2 3068; GFX10-W32-NEXT: s_mov_b32 s27, s9 3069; GFX10-W32-NEXT: s_mov_b32 s26, s8 3070; GFX10-W32-NEXT: s_mov_b32 s25, s7 3071; GFX10-W32-NEXT: s_mov_b32 s24, s6 3072; GFX10-W32-NEXT: s_mov_b32 s18, s16 3073; GFX10-W32-NEXT: s_mov_b32 s17, s15 3074; GFX10-W32-NEXT: s_mov_b32 s16, s14 3075; GFX10-W32-NEXT: s_mov_b32 s15, s13 3076; GFX10-W32-NEXT: s_mov_b32 s14, s12 3077; GFX10-W32-NEXT: s_mov_b32 s13, s11 3078; GFX10-W32-NEXT: s_mov_b32 s12, s10 3079; GFX10-W32-NEXT: v_mov_b32_e32 v1, s1 3080; GFX10-W32-NEXT: s_mov_b32 exec_lo, s29 3081; GFX10-W32-NEXT: buffer_store_dword v0, v1, s[20:23], 0 idxen 3082; GFX10-W32-NEXT: s_mov_b32 s1, exec_lo 3083; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 3084; GFX10-W32-NEXT: buffer_load_dword v2, v1, s[20:23], 0 idxen 3085; GFX10-W32-NEXT: s_mov_b32 exec_lo, s1 3086; GFX10-W32-NEXT: s_or_saveexec_b32 s1, -1 3087; GFX10-W32-NEXT: v_mov_b32_e32 v3, s0 3088; GFX10-W32-NEXT: buffer_load_dword v3, v3, s[24:27], 0 idxen 3089; GFX10-W32-NEXT: s_mov_b32 exec_lo, s1 3090; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo 3091; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 3092; GFX10-W32-NEXT: s_waitcnt vmcnt(1) 3093; GFX10-W32-NEXT: v_add_f32_e32 v2, v2, v2 3094; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 3095; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 3096; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2 3097; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 3098; GFX10-W32-NEXT: v_mov_b32_e32 v4, v3 3099; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v0 3100; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v4 3101; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s28 3102; GFX10-W32-NEXT: image_sample v0, v0, s[12:19], s[20:23] dmask:0x1 dim:SQ_RSRC_IMG_1D 3103; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 3104; GFX10-W32-NEXT: buffer_store_dword v0, v1, s[20:23], 0 idxen 3105; GFX10-W32-NEXT: buffer_load_dword v0, v1, s[20:23], 0 idxen 3106; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 3107; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 3108; GFX10-W32-NEXT: ; return to shader part epilog 3109main_body: 3110 call void @llvm.amdgcn.struct.buffer.store.f32(float %inp, <4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0) 3111 %reload = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0) 3112 %temp = fadd float %reload, %reload 3113 %temp2 = call float @llvm.amdgcn.strict.wqm.f32(float %temp) 3114 %temp3 = fadd float %temp2, %temp2 3115 %reload_wwm = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res2, i32 %idx0, i32 0, i32 0, i32 0) 3116 %temp4 = call float @llvm.amdgcn.strict.wwm.f32(float %reload_wwm) 3117 %temp5 = fadd float %temp3, %temp4 3118 %tex = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %temp5, <8 x i32> %res3, <4 x i32> %res, i1 false, i32 0, i32 0) 3119 call void @llvm.amdgcn.struct.buffer.store.f32(float %tex, <4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0) 3120 %out = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0) 3121 ret float %out 3122} 3123 3124define amdgpu_ps float @test_strict_wwm_strict_wqm_wqm(i32 inreg %idx0, i32 inreg %idx1, <4 x i32> inreg %res, float %inp, <8 x i32> inreg %res2) { 3125; GFX9-W64-LABEL: test_strict_wwm_strict_wqm_wqm: 3126; GFX9-W64: ; %bb.0: ; %main_body 3127; GFX9-W64-NEXT: s_mov_b64 s[20:21], exec 3128; GFX9-W64-NEXT: s_mov_b32 s15, s13 3129; GFX9-W64-NEXT: s_mov_b64 s[22:23], exec 3130; GFX9-W64-NEXT: s_wqm_b64 exec, exec 3131; GFX9-W64-NEXT: s_mov_b32 s19, s5 3132; GFX9-W64-NEXT: s_mov_b32 s18, s4 3133; GFX9-W64-NEXT: s_mov_b32 s17, s3 3134; GFX9-W64-NEXT: s_mov_b32 s16, s2 3135; GFX9-W64-NEXT: s_mov_b32 s14, s12 3136; GFX9-W64-NEXT: s_mov_b32 s13, s11 3137; GFX9-W64-NEXT: s_mov_b32 s12, s10 3138; GFX9-W64-NEXT: s_mov_b32 s11, s9 3139; GFX9-W64-NEXT: s_mov_b32 s10, s8 3140; GFX9-W64-NEXT: s_mov_b32 s9, s7 3141; GFX9-W64-NEXT: s_mov_b32 s8, s6 3142; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 3143; GFX9-W64-NEXT: s_mov_b64 exec, s[22:23] 3144; GFX9-W64-NEXT: buffer_store_dword v0, v1, s[16:19], 0 idxen 3145; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1 3146; GFX9-W64-NEXT: v_mov_b32_e32 v2, s1 3147; GFX9-W64-NEXT: buffer_load_dword v2, v2, s[16:19], 0 idxen 3148; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3] 3149; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec 3150; GFX9-W64-NEXT: s_wqm_b64 exec, exec 3151; GFX9-W64-NEXT: buffer_load_dword v3, v1, s[16:19], 0 idxen 3152; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1] 3153; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], -1 3154; GFX9-W64-NEXT: s_waitcnt vmcnt(1) 3155; GFX9-W64-NEXT: v_add_f32_e32 v2, v2, v2 3156; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1] 3157; GFX9-W64-NEXT: s_wqm_b64 exec, exec 3158; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2 3159; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v0 3160; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 3161; GFX9-W64-NEXT: v_mov_b32_e32 v4, v3 3162; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v4 3163; GFX9-W64-NEXT: s_and_b64 exec, exec, s[20:21] 3164; GFX9-W64-NEXT: image_sample v0, v0, s[8:15], s[16:19] dmask:0x1 3165; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 3166; GFX9-W64-NEXT: buffer_store_dword v0, v1, s[16:19], 0 idxen 3167; GFX9-W64-NEXT: buffer_load_dword v0, v1, s[16:19], 0 idxen 3168; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 3169; GFX9-W64-NEXT: ; return to shader part epilog 3170; 3171; GFX10-W32-LABEL: test_strict_wwm_strict_wqm_wqm: 3172; GFX10-W32: ; %bb.0: ; %main_body 3173; GFX10-W32-NEXT: s_mov_b32 s20, exec_lo 3174; GFX10-W32-NEXT: s_mov_b32 s15, s13 3175; GFX10-W32-NEXT: s_mov_b32 s21, exec_lo 3176; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 3177; GFX10-W32-NEXT: s_mov_b32 s19, s5 3178; GFX10-W32-NEXT: s_mov_b32 s18, s4 3179; GFX10-W32-NEXT: s_mov_b32 s17, s3 3180; GFX10-W32-NEXT: s_mov_b32 s16, s2 3181; GFX10-W32-NEXT: s_mov_b32 s14, s12 3182; GFX10-W32-NEXT: s_mov_b32 s13, s11 3183; GFX10-W32-NEXT: s_mov_b32 s12, s10 3184; GFX10-W32-NEXT: s_mov_b32 s11, s9 3185; GFX10-W32-NEXT: s_mov_b32 s10, s8 3186; GFX10-W32-NEXT: s_mov_b32 s9, s7 3187; GFX10-W32-NEXT: s_mov_b32 s8, s6 3188; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 3189; GFX10-W32-NEXT: s_mov_b32 exec_lo, s21 3190; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 3191; GFX10-W32-NEXT: v_mov_b32_e32 v2, s1 3192; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 3193; GFX10-W32-NEXT: buffer_store_dword v0, v1, s[16:19], 0 idxen 3194; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 3195; GFX10-W32-NEXT: buffer_load_dword v2, v2, s[16:19], 0 idxen 3196; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 3197; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo 3198; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 3199; GFX10-W32-NEXT: buffer_load_dword v3, v1, s[16:19], 0 idxen 3200; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 3201; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1 3202; GFX10-W32-NEXT: s_waitcnt vmcnt(1) 3203; GFX10-W32-NEXT: v_add_f32_e32 v2, v2, v2 3204; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 3205; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 3206; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2 3207; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 3208; GFX10-W32-NEXT: v_mov_b32_e32 v4, v3 3209; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v0 3210; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v4 3211; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s20 3212; GFX10-W32-NEXT: image_sample v0, v0, s[8:15], s[16:19] dmask:0x1 dim:SQ_RSRC_IMG_1D 3213; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 3214; GFX10-W32-NEXT: buffer_store_dword v0, v1, s[16:19], 0 idxen 3215; GFX10-W32-NEXT: buffer_load_dword v0, v1, s[16:19], 0 idxen 3216; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 3217; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 3218; GFX10-W32-NEXT: ; return to shader part epilog 3219main_body: 3220 call void @llvm.amdgcn.struct.buffer.store.f32(float %inp, <4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0) 3221 %reload = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0) 3222 %temp = fadd float %reload, %reload 3223 %temp2 = call float @llvm.amdgcn.strict.wwm.f32(float %temp) 3224 %temp3 = fadd float %temp2, %temp2 3225 %reload_wwm = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0) 3226 %temp4 = call float @llvm.amdgcn.strict.wqm.f32(float %reload_wwm) 3227 %temp5 = fadd float %temp3, %temp4 3228 %tex = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %temp5, <8 x i32> %res2, <4 x i32> %res, i1 false, i32 0, i32 0) 3229 call void @llvm.amdgcn.struct.buffer.store.f32(float %tex, <4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0) 3230 %out = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0) 3231 ret float %out 3232} 3233 3234;TODO: WQM -> StrictWQM transition could be improved. StrictWQM could use the exec from the previous state instead of calling s_wqm again. 3235define amdgpu_ps float @test_wqm_strict_wqm_wqm(i32 inreg %idx0, i32 inreg %idx1, <4 x i32> inreg %res, float %inp, <8 x i32> inreg %res2) { 3236; GFX9-W64-LABEL: test_wqm_strict_wqm_wqm: 3237; GFX9-W64: ; %bb.0: ; %main_body 3238; GFX9-W64-NEXT: s_mov_b64 s[20:21], exec 3239; GFX9-W64-NEXT: s_mov_b32 s15, s13 3240; GFX9-W64-NEXT: s_mov_b64 s[22:23], exec 3241; GFX9-W64-NEXT: s_wqm_b64 exec, exec 3242; GFX9-W64-NEXT: s_mov_b32 s19, s5 3243; GFX9-W64-NEXT: s_mov_b32 s18, s4 3244; GFX9-W64-NEXT: s_mov_b32 s17, s3 3245; GFX9-W64-NEXT: s_mov_b32 s16, s2 3246; GFX9-W64-NEXT: s_mov_b32 s14, s12 3247; GFX9-W64-NEXT: s_mov_b32 s13, s11 3248; GFX9-W64-NEXT: s_mov_b32 s12, s10 3249; GFX9-W64-NEXT: s_mov_b32 s11, s9 3250; GFX9-W64-NEXT: s_mov_b32 s10, s8 3251; GFX9-W64-NEXT: s_mov_b32 s9, s7 3252; GFX9-W64-NEXT: s_mov_b32 s8, s6 3253; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0 3254; GFX9-W64-NEXT: s_mov_b64 exec, s[22:23] 3255; GFX9-W64-NEXT: buffer_store_dword v0, v1, s[16:19], 0 idxen 3256; GFX9-W64-NEXT: s_wqm_b64 exec, exec 3257; GFX9-W64-NEXT: v_mov_b32_e32 v0, s1 3258; GFX9-W64-NEXT: buffer_load_dword v0, v0, s[16:19], 0 idxen 3259; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec 3260; GFX9-W64-NEXT: s_wqm_b64 exec, exec 3261; GFX9-W64-NEXT: buffer_load_dword v2, v1, s[16:19], 0 idxen 3262; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1] 3263; GFX9-W64-NEXT: s_waitcnt vmcnt(1) 3264; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v0 3265; GFX9-W64-NEXT: image_sample v0, v0, s[8:15], s[16:19] dmask:0x1 3266; GFX9-W64-NEXT: s_waitcnt vmcnt(1) 3267; GFX9-W64-NEXT: v_mov_b32_e32 v3, v2 3268; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 3269; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v0 3270; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v3 3271; GFX9-W64-NEXT: s_and_b64 exec, exec, s[20:21] 3272; GFX9-W64-NEXT: image_sample v0, v0, s[8:15], s[16:19] dmask:0x1 3273; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 3274; GFX9-W64-NEXT: buffer_store_dword v0, v1, s[16:19], 0 idxen 3275; GFX9-W64-NEXT: buffer_load_dword v0, v1, s[16:19], 0 idxen 3276; GFX9-W64-NEXT: s_waitcnt vmcnt(0) 3277; GFX9-W64-NEXT: ; return to shader part epilog 3278; 3279; GFX10-W32-LABEL: test_wqm_strict_wqm_wqm: 3280; GFX10-W32: ; %bb.0: ; %main_body 3281; GFX10-W32-NEXT: s_mov_b32 s20, exec_lo 3282; GFX10-W32-NEXT: s_mov_b32 s15, s13 3283; GFX10-W32-NEXT: s_mov_b32 s21, exec_lo 3284; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 3285; GFX10-W32-NEXT: s_mov_b32 s19, s5 3286; GFX10-W32-NEXT: s_mov_b32 s18, s4 3287; GFX10-W32-NEXT: s_mov_b32 s17, s3 3288; GFX10-W32-NEXT: s_mov_b32 s16, s2 3289; GFX10-W32-NEXT: s_mov_b32 s14, s12 3290; GFX10-W32-NEXT: s_mov_b32 s13, s11 3291; GFX10-W32-NEXT: s_mov_b32 s12, s10 3292; GFX10-W32-NEXT: s_mov_b32 s11, s9 3293; GFX10-W32-NEXT: s_mov_b32 s10, s8 3294; GFX10-W32-NEXT: s_mov_b32 s9, s7 3295; GFX10-W32-NEXT: s_mov_b32 s8, s6 3296; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0 3297; GFX10-W32-NEXT: s_mov_b32 exec_lo, s21 3298; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 3299; GFX10-W32-NEXT: v_mov_b32_e32 v3, s1 3300; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s20 3301; GFX10-W32-NEXT: buffer_store_dword v0, v1, s[16:19], 0 idxen 3302; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 3303; GFX10-W32-NEXT: buffer_load_dword v0, v3, s[16:19], 0 idxen 3304; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo 3305; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo 3306; GFX10-W32-NEXT: buffer_load_dword v2, v1, s[16:19], 0 idxen 3307; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0 3308; GFX10-W32-NEXT: s_waitcnt vmcnt(1) 3309; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v0 3310; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 3311; GFX10-W32-NEXT: v_mov_b32_e32 v3, v2 3312; GFX10-W32-NEXT: image_sample v0, v0, s[8:15], s[16:19] dmask:0x1 dim:SQ_RSRC_IMG_1D 3313; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 3314; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v0 3315; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v3 3316; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s20 3317; GFX10-W32-NEXT: image_sample v0, v0, s[8:15], s[16:19] dmask:0x1 dim:SQ_RSRC_IMG_1D 3318; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 3319; GFX10-W32-NEXT: buffer_store_dword v0, v1, s[16:19], 0 idxen 3320; GFX10-W32-NEXT: buffer_load_dword v0, v1, s[16:19], 0 idxen 3321; GFX10-W32-NEXT: s_waitcnt vmcnt(0) 3322; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x0 3323; GFX10-W32-NEXT: ; return to shader part epilog 3324main_body: 3325 call void @llvm.amdgcn.struct.buffer.store.f32(float %inp, <4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0) 3326 %reload = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0) 3327 %temp = fadd float %reload, %reload 3328 %tex = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %temp, <8 x i32> %res2, <4 x i32> %res, i1 false, i32 0, i32 0) 3329 %temp2 = fadd float %tex, %tex 3330 %reload_wwm = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0) 3331 %temp3 = call float @llvm.amdgcn.strict.wqm.f32(float %reload_wwm) 3332 %temp4 = fadd float %temp2, %temp3 3333 %tex2 = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %temp4, <8 x i32> %res2, <4 x i32> %res, i1 false, i32 0, i32 0) 3334 call void @llvm.amdgcn.struct.buffer.store.f32(float %tex2, <4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0) 3335 %out = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0) 3336 ret float %out 3337} 3338 3339; Check if the correct VCC register is selected. WQM pass incorrectly uses VCC for 3340; vector comparisons in Wave32 mode. 3341define amdgpu_ps void @test_for_deactivating_lanes_in_wave32(float addrspace(6)* inreg %0) { 3342; GFX9-W64-LABEL: test_for_deactivating_lanes_in_wave32: 3343; GFX9-W64: ; %bb.0: ; %main_body 3344; GFX9-W64-NEXT: s_mov_b32 s3, 0x31016fac 3345; GFX9-W64-NEXT: s_mov_b32 s2, 32 3346; GFX9-W64-NEXT: s_mov_b32 s1, 0x8000 3347; GFX9-W64-NEXT: s_buffer_load_dword s0, s[0:3], 0x0 3348; GFX9-W64-NEXT: s_waitcnt lgkmcnt(0) 3349; GFX9-W64-NEXT: v_cmp_le_f32_e64 vcc, s0, 0 3350; GFX9-W64-NEXT: s_andn2_b64 s[4:5], exec, vcc 3351; GFX9-W64-NEXT: s_cbranch_scc0 .LBB50_1 3352; GFX9-W64-NEXT: s_endpgm 3353; GFX9-W64-NEXT: .LBB50_1: 3354; GFX9-W64-NEXT: s_mov_b64 exec, 0 3355; GFX9-W64-NEXT: exp null off, off, off, off done vm 3356; GFX9-W64-NEXT: s_endpgm 3357; 3358; GFX10-W32-LABEL: test_for_deactivating_lanes_in_wave32: 3359; GFX10-W32: ; %bb.0: ; %main_body 3360; GFX10-W32-NEXT: s_mov_b32 s3, 0x31016fac 3361; GFX10-W32-NEXT: s_mov_b32 s2, 32 3362; GFX10-W32-NEXT: s_mov_b32 s1, 0x8000 3363; GFX10-W32-NEXT: s_buffer_load_dword s0, s[0:3], 0x0 3364; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0) 3365; GFX10-W32-NEXT: v_cmp_le_f32_e64 vcc_lo, s0, 0 3366; GFX10-W32-NEXT: s_andn2_b32 s4, exec_lo, vcc_lo 3367; GFX10-W32-NEXT: s_cbranch_scc0 .LBB50_1 3368; GFX10-W32-NEXT: s_endpgm 3369; GFX10-W32-NEXT: .LBB50_1: 3370; GFX10-W32-NEXT: s_mov_b32 exec_lo, 0 3371; GFX10-W32-NEXT: exp null off, off, off, off done vm 3372; GFX10-W32-NEXT: s_endpgm 3373main_body: 3374 %1 = ptrtoint float addrspace(6)* %0 to i32 3375 %2 = insertelement <4 x i32> <i32 poison, i32 32768, i32 32, i32 822177708>, i32 %1, i32 0 3376 %3 = call nsz arcp float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %2, i32 0, i32 0) #3 3377 %4 = fcmp nsz arcp ugt float %3, 0.000000e+00 3378 call void @llvm.amdgcn.kill(i1 %4) #1 3379 ret void 3380} 3381 3382declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #1 3383declare void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float>, i32, i32, <8 x i32>, i32, i32) #1 3384 3385declare void @llvm.amdgcn.struct.buffer.store.f32(float, <4 x i32>, i32, i32, i32, i32 immarg) #2 3386declare void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32, i32 immarg) #2 3387declare void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32 immarg) #2 3388declare void @llvm.amdgcn.raw.buffer.store.f32(float, <4 x i32>, i32, i32, i32 immarg) #2 3389declare float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32>, i32, i32, i32) #3 3390declare float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32>, i32, i32, i32, i32) #3 3391 3392declare <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32, i32, <8 x i32>, i32, i32) #3 3393declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #3 3394declare <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #3 3395declare float @llvm.amdgcn.image.sample.1d.f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #3 3396declare void @llvm.amdgcn.kill(i1) #1 3397declare float @llvm.amdgcn.wqm.f32(float) #3 3398declare i32 @llvm.amdgcn.wqm.i32(i32) #3 3399declare float @llvm.amdgcn.strict.wwm.f32(float) #3 3400declare i32 @llvm.amdgcn.strict.wwm.i32(i32) #3 3401declare float @llvm.amdgcn.wwm.f32(float) #3 3402declare i32 @llvm.amdgcn.wwm.i32(i32) #3 3403declare float @llvm.amdgcn.strict.wqm.f32(float) #3 3404declare i32 @llvm.amdgcn.strict.wqm.i32(i32) #3 3405declare i32 @llvm.amdgcn.set.inactive.i32(i32, i32) #4 3406declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #3 3407declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32) #3 3408declare <2 x half> @llvm.amdgcn.cvt.pkrtz(float, float) #3 3409declare void @llvm.amdgcn.exp.compr.v2f16(i32, i32, <2 x half>, <2 x half>, i1, i1) #1 3410declare float @llvm.amdgcn.interp.p1(float, i32, i32, i32) #2 3411declare float @llvm.amdgcn.interp.p2(float, float, i32, i32, i32) #2 3412declare i32 @llvm.amdgcn.ds.swizzle(i32, i32) 3413declare float @llvm.amdgcn.s.buffer.load.f32(<4 x i32>, i32, i32 immarg) #7 3414 3415attributes #1 = { nounwind } 3416attributes #2 = { nounwind readonly } 3417attributes #3 = { nounwind readnone } 3418attributes #4 = { nounwind readnone convergent } 3419attributes #5 = { "amdgpu-ps-wqm-outputs" } 3420attributes #6 = { nounwind "InitialPSInputAddr"="2" } 3421attributes #7 = { nounwind readnone willreturn } 3422