1; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=CHECK %s 2; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=CHECK %s 3 4; Check that WQM isn't triggered by image load/store intrinsics. 5; 6;CHECK-LABEL: {{^}}test1: 7;CHECK-NOT: s_wqm 8define amdgpu_ps <4 x float> @test1(<8 x i32> inreg %rsrc, i32 %c) { 9main_body: 10 %tex = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %c, <8 x i32> %rsrc, i32 0, i32 0) 11 call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> %tex, i32 15, i32 %c, <8 x i32> %rsrc, i32 0, i32 0) 12 ret <4 x float> %tex 13} 14 15; Check that WQM is triggered by code calculating inputs to image samples and is disabled as soon as possible 16; 17;CHECK-LABEL: {{^}}test2: 18;CHECK-NEXT: ; %main_body 19;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 20;CHECK-NEXT: s_wqm_b64 exec, exec 21;CHECK: interp 22;CHECK: s_and_b64 exec, exec, [[ORIG]] 23;CHECK-NOT: interp 24;CHECK: image_sample 25;CHECK-NOT: exec 26;CHECK: .size test2 27define amdgpu_ps <4 x float> @test2(i32 inreg, i32 inreg, i32 inreg, i32 inreg %m0, <8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, <2 x float> %pos) #6 { 28main_body: 29 %inst23 = extractelement <2 x float> %pos, i32 0 30 %inst24 = extractelement <2 x float> %pos, i32 1 31 %inst25 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 0, i32 0, i32 %m0) 32 %inst26 = tail call float @llvm.amdgcn.interp.p2(float %inst25, float %inst24, i32 0, i32 0, i32 %m0) 33 %inst28 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 1, i32 0, i32 %m0) 34 %inst29 = tail call float @llvm.amdgcn.interp.p2(float %inst28, float %inst24, i32 1, i32 0, i32 %m0) 35 %tex = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32 15, float %inst26, float %inst29, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 36 ret <4 x float> %tex 37} 38 39; ... but disabled for stores (and, in this simple case, not re-enabled) ... 40; 41;CHECK-LABEL: {{^}}test3: 42;CHECK-NEXT: ; %main_body 43;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 44;CHECK-NEXT: s_wqm_b64 exec, exec 45;CHECK: s_and_b64 exec, exec, [[ORIG]] 46;CHECK: image_sample 47;CHECK: store 48;CHECK-NOT: exec 49;CHECK: .size test3 50define amdgpu_ps <4 x float> @test3(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float %c) { 51main_body: 52 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 53 %tex.1 = bitcast <4 x float> %tex to <4 x i32> 54 %tex.2 = extractelement <4 x i32> %tex.1, i32 0 55 56 call void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float> %tex, <4 x i32> undef, i32 %tex.2, i32 0, i32 0, i32 0) 57 58 ret <4 x float> %tex 59} 60 61; ... and disabled for export. 62; 63;CHECK-LABEL: {{^}}test3x: 64;CHECK-NEXT: ; %main_body 65;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 66;CHECK-NEXT: s_wqm_b64 exec, exec 67;CHECK: s_and_b64 exec, exec, [[ORIG]] 68;CHECK: image_sample 69;CHECK: exp 70;CHECK-NOT: exec 71;CHECK: .size test3x 72define amdgpu_ps void @test3x(i32 inreg, i32 inreg, i32 inreg, i32 inreg %m0, <8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, <2 x float> %pos) #6 { 73main_body: 74 %inst23 = extractelement <2 x float> %pos, i32 0 75 %inst24 = extractelement <2 x float> %pos, i32 1 76 %inst25 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 0, i32 0, i32 %m0) 77 %inst26 = tail call float @llvm.amdgcn.interp.p2(float %inst25, float %inst24, i32 0, i32 0, i32 %m0) 78 %inst28 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 1, i32 0, i32 %m0) 79 %inst29 = tail call float @llvm.amdgcn.interp.p2(float %inst28, float %inst24, i32 1, i32 0, i32 %m0) 80 %tex = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32 15, float %inst26, float %inst29, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 81 %tex.0 = extractelement <4 x float> %tex, i32 0 82 %tex.1 = extractelement <4 x float> %tex, i32 1 83 %tex.2 = extractelement <4 x float> %tex, i32 2 84 %tex.3 = extractelement <4 x float> %tex, i32 3 85 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %tex.0, float %tex.1, float %tex.2, float %tex.3, i1 true, i1 true) 86 ret void 87} 88 89; Check that WQM is re-enabled when required. 90; 91;CHECK-LABEL: {{^}}test4: 92;CHECK-NEXT: ; %main_body 93;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 94;CHECK-NEXT: s_wqm_b64 exec, exec 95;CHECK: v_mul_lo_u32 [[MUL:v[0-9]+]], v0, v1 96;CHECK: image_sample 97;CHECK: s_and_b64 exec, exec, [[ORIG]] 98;CHECK: image_sample 99;CHECK: store 100define amdgpu_ps <4 x float> @test4(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float addrspace(1)* inreg %ptr, i32 %c, i32 %d, float %data) { 101main_body: 102 %c.1 = mul i32 %c, %d 103 104 call void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float> undef, <4 x i32> undef, i32 %c.1, i32 0, i32 0, i32 0) 105 %c.1.bc = bitcast i32 %c.1 to float 106 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.1.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 107 %tex0 = extractelement <4 x float> %tex, i32 0 108 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 109 ret <4 x float> %dtex 110} 111 112; Check that WQM is triggered by the wqm intrinsic. 113; 114;CHECK-LABEL: {{^}}test5: 115;CHECK: s_wqm_b64 exec, exec 116;CHECK: buffer_load_dword 117;CHECK: buffer_load_dword 118;CHECK: v_add_f32_e32 119; WQM was inserting an unecessary v_mov to self after the v_add. Make sure this 120; does not happen - the v_add should write the return reg directly. 121;CHECK-NOT: v_mov_b32_e32 122define amdgpu_ps float @test5(i32 inreg %idx0, i32 inreg %idx1) { 123main_body: 124 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 125 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 126 %out = fadd float %src0, %src1 127 %out.0 = call float @llvm.amdgcn.wqm.f32(float %out) 128 ret float %out.0 129} 130 131; Check that the wqm intrinsic works correctly for integers. 132; 133;CHECK-LABEL: {{^}}test6: 134;CHECK: s_wqm_b64 exec, exec 135;CHECK: buffer_load_dword 136;CHECK: buffer_load_dword 137;CHECK: v_add_f32_e32 138define amdgpu_ps float @test6(i32 inreg %idx0, i32 inreg %idx1) { 139main_body: 140 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 141 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 142 %out = fadd float %src0, %src1 143 %out.0 = bitcast float %out to i32 144 %out.1 = call i32 @llvm.amdgcn.wqm.i32(i32 %out.0) 145 %out.2 = bitcast i32 %out.1 to float 146 ret float %out.2 147} 148 149; NOTE: llvm.amdgcn.wwm is deprecated, use llvm.amdgcn.strict.wwm instead. 150 151; Check that WWM is triggered by the wwm intrinsic. 152; 153;CHECK-LABEL: {{^}}test_wwm1: 154;CHECK: s_or_saveexec_b64 s{{\[[0-9]+:[0-9]+\]}}, -1 155;CHECK: buffer_load_dword 156;CHECK: buffer_load_dword 157;CHECK: v_add_f32_e32 158define amdgpu_ps float @test_wwm1(i32 inreg %idx0, i32 inreg %idx1) { 159main_body: 160 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 161 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 162 %out = fadd float %src0, %src1 163 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out) 164 ret float %out.0 165} 166 167; Same as above, but with an integer type. 168; 169;CHECK-LABEL: {{^}}test_wwm2: 170;CHECK: s_or_saveexec_b64 s{{\[[0-9]+:[0-9]+\]}}, -1 171;CHECK: buffer_load_dword 172;CHECK: buffer_load_dword 173;CHECK: v_add_{{[iu]}}32_e32 174define amdgpu_ps float @test_wwm2(i32 inreg %idx0, i32 inreg %idx1) { 175main_body: 176 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 177 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 178 %src0.0 = bitcast float %src0 to i32 179 %src1.0 = bitcast float %src1 to i32 180 %out = add i32 %src0.0, %src1.0 181 %out.0 = call i32 @llvm.amdgcn.wwm.i32(i32 %out) 182 %out.1 = bitcast i32 %out.0 to float 183 ret float %out.1 184} 185 186; Check that we don't leave WWM on for computations that don't require WWM, 187; since that will lead clobbering things that aren't supposed to be clobbered 188; in cases like this. 189; We enforce this by checking that v_add gets emitted in the same block as 190; WWM computations. 191; 192;CHECK-LABEL: {{^}}test_wwm3: 193;CHECK: %if 194;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1 195;CHECK: buffer_load_dword 196;CHECK: v_add_f32_e32 197;CHECK: s_mov_b64 exec, [[ORIG]] 198;CHECK: v_add_f32_e32 199;CHECK: %endif 200define amdgpu_ps float @test_wwm3(i32 inreg %idx) { 201main_body: 202 ; use mbcnt to make sure the branch is divergent 203 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 204 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 205 %cc = icmp uge i32 %hi, 32 206 br i1 %cc, label %endif, label %if 207 208if: 209 %src = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 210 %out = fadd float %src, %src 211 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out) 212 %out.1 = fadd float %src, %out.0 213 br label %endif 214 215endif: 216 %out.2 = phi float [ %out.1, %if ], [ 0.0, %main_body ] 217 ret float %out.2 218} 219 220; Check that WWM writes aren't coalesced with non-WWM writes, since the WWM 221; write could clobber disabled channels in the non-WWM one. 222; We enforce this by checking that v_mov gets emitted in the same block as 223; WWM computations. 224; 225;CHECK-LABEL: {{^}}test_wwm4: 226;CHECK: %if 227;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1 228;CHECK: buffer_load_dword 229;CHECK: v_add_f32_e32 230;CHECK: s_mov_b64 exec, [[ORIG]] 231;CHECK-NEXT: v_mov_b32_e32 232;CHECK: %endif 233define amdgpu_ps float @test_wwm4(i32 inreg %idx) { 234main_body: 235 ; use mbcnt to make sure the branch is divergent 236 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 237 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 238 %cc = icmp uge i32 %hi, 32 239 br i1 %cc, label %endif, label %if 240 241if: 242 %src = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 243 %out = fadd float %src, %src 244 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out) 245 br label %endif 246 247endif: 248 %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ] 249 ret float %out.1 250} 251 252; Make sure the transition from Exact to WWM then WQM works properly. 253; 254;CHECK-LABEL: {{^}}test_wwm5: 255;CHECK: buffer_load_dword 256;CHECK: buffer_store_dword 257;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1 258;CHECK: buffer_load_dword 259;CHECK: v_add_f32_e32 260;CHECK: s_mov_b64 exec, [[ORIG]] 261;CHECK: s_wqm_b64 exec, exec 262define amdgpu_ps float @test_wwm5(i32 inreg %idx0, i32 inreg %idx1) { 263main_body: 264 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 265 call void @llvm.amdgcn.struct.buffer.store.f32(float %src0, <4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 266 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 267 %temp = fadd float %src1, %src1 268 %temp.0 = call float @llvm.amdgcn.wwm.f32(float %temp) 269 %out = fadd float %temp.0, %temp.0 270 %out.0 = call float @llvm.amdgcn.wqm.f32(float %out) 271 ret float %out.0 272} 273 274; Check that WWM is turned on correctly across basic block boundaries. 275; if..then..endif version 276; 277;CHECK-LABEL: {{^}}test_wwm6_then: 278;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1 279;SI-CHECK: buffer_load_dword 280;VI-CHECK: flat_load_dword 281;CHECK: s_mov_b64 exec, [[ORIG]] 282;CHECK: %if 283;CHECK: s_or_saveexec_b64 [[ORIG2:s\[[0-9]+:[0-9]+\]]], -1 284;SI-CHECK: buffer_load_dword 285;VI-CHECK: flat_load_dword 286;CHECK: v_add_f32_e32 287;CHECK: s_mov_b64 exec, [[ORIG2]] 288;CHECK: %endif 289define amdgpu_ps float @test_wwm6_then() { 290main_body: 291 %src0 = load volatile float, float addrspace(1)* undef 292 ; use mbcnt to make sure the branch is divergent 293 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 294 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 295 %cc = icmp uge i32 %hi, 32 296 br i1 %cc, label %endif, label %if 297 298if: 299 %src1 = load volatile float, float addrspace(1)* undef 300 %out = fadd float %src0, %src1 301 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out) 302 br label %endif 303 304endif: 305 %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ] 306 ret float %out.1 307} 308 309; Check that WWM is turned on correctly across basic block boundaries. 310; loop version 311; 312;CHECK-LABEL: {{^}}test_wwm6_loop: 313;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1 314;SI-CHECK: buffer_load_dword 315;VI-CHECK: flat_load_dword 316;CHECK: s_mov_b64 exec, [[ORIG]] 317;CHECK: %loop 318;CHECK: s_or_saveexec_b64 [[ORIG2:s\[[0-9]+:[0-9]+\]]], -1 319;SI-CHECK: buffer_load_dword 320;VI-CHECK: flat_load_dword 321;CHECK: s_mov_b64 exec, [[ORIG2]] 322;CHECK: %endloop 323define amdgpu_ps float @test_wwm6_loop() { 324main_body: 325 %src0 = load volatile float, float addrspace(1)* undef 326 ; use mbcnt to make sure the branch is divergent 327 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 328 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 329 br label %loop 330 331loop: 332 %counter = phi i32 [ %lo, %main_body ], [ %counter.1, %loop ] 333 %src1 = load volatile float, float addrspace(1)* undef 334 %out = fadd float %src0, %src1 335 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out) 336 %counter.1 = sub i32 %counter, 1 337 %cc = icmp ne i32 %counter.1, 0 338 br i1 %cc, label %loop, label %endloop 339 340endloop: 341 ret float %out.0 342} 343 344; Check that @llvm.amdgcn.set.inactive disables WWM. 345; 346;CHECK-LABEL: {{^}}test_wwm_set_inactive1: 347;CHECK: buffer_load_dword 348;CHECK: s_not_b64 exec, exec 349;CHECK: v_mov_b32_e32 350;CHECK: s_not_b64 exec, exec 351;CHECK: s_or_saveexec_b64 s{{\[[0-9]+:[0-9]+\]}}, -1 352;CHECK: v_add_{{[iu]}}32_e32 353define amdgpu_ps void @test_wwm_set_inactive1(i32 inreg %idx) { 354main_body: 355 %src = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 356 %src.0 = bitcast float %src to i32 357 %src.1 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %src.0, i32 0) 358 %out = add i32 %src.1, %src.1 359 %out.0 = call i32 @llvm.amdgcn.wwm.i32(i32 %out) 360 %out.1 = bitcast i32 %out.0 to float 361 call void @llvm.amdgcn.struct.buffer.store.f32(float %out.1, <4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 362 ret void 363} 364 365; Check that Strict WQM is triggered by the strict_wqm intrinsic. 366; 367;CHECK-LABEL: {{^}}test_strict_wqm1: 368;CHECK: s_mov_b64 s{{\[[0-9]+:[0-9]+\]}}, exec 369;CHECK: s_wqm_b64 exec, exec 370;CHECK: buffer_load_dword 371;CHECK: buffer_load_dword 372;CHECK: v_add_f32_e32 373define amdgpu_ps float @test_strict_wqm1(i32 inreg %idx0, i32 inreg %idx1) { 374main_body: 375 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 376 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 377 %out = fadd float %src0, %src1 378 %out.0 = call float @llvm.amdgcn.strict.wqm.f32(float %out) 379 ret float %out.0 380} 381 382; Same as above, but with an integer type. 383; 384;CHECK-LABEL: {{^}}test_strict_wqm2: 385;CHECK: s_mov_b64 s{{\[[0-9]+:[0-9]+\]}}, exec 386;CHECK: s_wqm_b64 exec, exec 387;CHECK: buffer_load_dword 388;CHECK: buffer_load_dword 389;CHECK: v_add_{{[iu]}}32_e32 390define amdgpu_ps float @test_strict_wqm2(i32 inreg %idx0, i32 inreg %idx1) { 391main_body: 392 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 393 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 394 %src0.0 = bitcast float %src0 to i32 395 %src1.0 = bitcast float %src1 to i32 396 %out = add i32 %src0.0, %src1.0 397 %out.0 = call i32 @llvm.amdgcn.strict.wqm.i32(i32 %out) 398 %out.1 = bitcast i32 %out.0 to float 399 ret float %out.1 400} 401 402; Check that we don't leave Strict WQM on for computations that don't require it, 403; since that will lead clobbering things that aren't supposed to be clobbered 404; in cases like this. 405; We enforce this by checking that v_add gets emitted in the same block as 406; WWM computations. 407; 408;CHECK-LABEL: {{^}}test_strict_wqm3: 409;CHECK: %if 410;CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 411;CHECK: s_wqm_b64 exec, exec 412;CHECK: buffer_load_dword 413;CHECK: v_add_f32_e32 414;CHECK: s_mov_b64 exec, [[ORIG]] 415;CHECK: v_add_f32_e32 416;CHECK: %endif 417define amdgpu_ps float @test_strict_wqm3(i32 inreg %idx) { 418main_body: 419 ; use mbcnt to make sure the branch is divergent 420 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 421 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 422 %cc = icmp uge i32 %hi, 32 423 br i1 %cc, label %endif, label %if 424 425if: 426 %src = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 427 %out = fadd float %src, %src 428 %out.0 = call float @llvm.amdgcn.strict.wqm.f32(float %out) 429 %out.1 = fadd float %src, %out.0 430 br label %endif 431 432endif: 433 %out.2 = phi float [ %out.1, %if ], [ 0.0, %main_body ] 434 ret float %out.2 435} 436 437; Check that Strict WQM writes aren't coalesced with non-strict writes, since 438; the Strict WQM write could clobber disabled channels in the non-strict one. 439; We enforce this by checking that v_mov gets emitted in the same block as 440; WWM computations. 441; 442;CHECK-LABEL: {{^}}test_strict_wqm4: 443;CHECK: %if 444;CHECK: s_mov_b64 s{{\[[0-9]+:[0-9]+\]}}, exec 445;CHECK: s_wqm_b64 exec, exec 446;CHECK: buffer_load_dword 447;CHECK: v_add_f32_e32 448;CHECK: s_mov_b64 exec, [[ORIG]] 449;CHECK-NEXT: v_mov_b32_e32 450;CHECK: %endif 451define amdgpu_ps float @test_strict_wqm4(i32 inreg %idx) { 452main_body: 453 ; use mbcnt to make sure the branch is divergent 454 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 455 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 456 %cc = icmp uge i32 %hi, 32 457 br i1 %cc, label %endif, label %if 458 459if: 460 %src = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 461 %out = fadd float %src, %src 462 %out.0 = call float @llvm.amdgcn.strict.wqm.f32(float %out) 463 br label %endif 464 465endif: 466 %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ] 467 ret float %out.1 468} 469 470; Make sure the transition from Exact to Strict WQM then WQM works properly. 471; 472;CHECK-LABEL: {{^}}test_strict_wqm5: 473;CHECK: buffer_load_dword 474;CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 475;CHECK: buffer_store_dword 476;CHECK: s_wqm_b64 exec, exec 477;CHECK: buffer_load_dword 478;CHECK: v_add_f32_e32 479;CHECK: s_mov_b64 exec, [[ORIG]] 480;CHECK: s_wqm_b64 exec, exec 481define amdgpu_ps float @test_strict_wqm5(i32 inreg %idx0, i32 inreg %idx1) { 482main_body: 483 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 484 call void @llvm.amdgcn.struct.buffer.store.f32(float %src0, <4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 485 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 486 %temp = fadd float %src1, %src1 487 %temp.0 = call float @llvm.amdgcn.strict.wqm.f32(float %temp) 488 %out = fadd float %temp.0, %temp.0 489 %out.0 = call float @llvm.amdgcn.wqm.f32(float %out) 490 ret float %out.0 491} 492 493; Check that Strict WQM is turned on correctly across basic block boundaries. 494; if..then..endif version 495; 496;CHECK-LABEL: {{^}}test_strict_wqm6_then: 497;CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 498;CHECK: s_wqm_b64 exec, exec 499;SI-CHECK: buffer_load_dword 500;VI-CHECK: flat_load_dword 501;CHECK: s_mov_b64 exec, [[ORIG]] 502;CHECK: %if 503;CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 504;CHECK: s_wqm_b64 exec, exec 505;SI-CHECK: buffer_load_dword 506;VI-CHECK: flat_load_dword 507;CHECK: v_add_f32_e32 508;CHECK: s_mov_b64 exec, [[ORIG2]] 509;CHECK: %endif 510define amdgpu_ps float @test_strict_wqm6_then() { 511main_body: 512 %src0 = load volatile float, float addrspace(1)* undef 513 ; use mbcnt to make sure the branch is divergent 514 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 515 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 516 %cc = icmp uge i32 %hi, 32 517 br i1 %cc, label %endif, label %if 518 519if: 520 %src1 = load volatile float, float addrspace(1)* undef 521 %out = fadd float %src0, %src1 522 %out.0 = call float @llvm.amdgcn.strict.wqm.f32(float %out) 523 br label %endif 524 525endif: 526 %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ] 527 ret float %out.1 528} 529 530; Check that Strict WQM is turned on correctly across basic block boundaries. 531; loop version 532; 533;CHECK-LABEL: {{^}}test_strict_wqm6_loop: 534;CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 535;CHECK: s_wqm_b64 exec, exec 536;SI-CHECK: buffer_load_dword 537;VI-CHECK: flat_load_dword 538;CHECK: s_mov_b64 exec, [[ORIG]] 539;CHECK: %loop 540;CHECK: s_mov_b64 [[ORIG2:s\[[0-9]+:[0-9]+\]]], exec 541;CHECK: s_wqm_b64 exec, exec 542;SI-CHECK: buffer_load_dword 543;VI-CHECK: flat_load_dword 544;CHECK: s_mov_b64 exec, [[ORIG2]] 545;CHECK: %endloop 546define amdgpu_ps float @test_strict_wqm6_loop() { 547main_body: 548 %src0 = load volatile float, float addrspace(1)* undef 549 ; use mbcnt to make sure the branch is divergent 550 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 551 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 552 br label %loop 553 554loop: 555 %counter = phi i32 [ %lo, %main_body ], [ %counter.1, %loop ] 556 %src1 = load volatile float, float addrspace(1)* undef 557 %out = fadd float %src0, %src1 558 %out.0 = call float @llvm.amdgcn.strict.wqm.f32(float %out) 559 %counter.1 = sub i32 %counter, 1 560 %cc = icmp ne i32 %counter.1, 0 561 br i1 %cc, label %loop, label %endloop 562 563endloop: 564 ret float %out.0 565} 566 567; Check that enabling WQM anywhere enables WQM for the set.inactive source. 568; 569;CHECK-LABEL: {{^}}test_set_inactive2: 570;CHECK: s_wqm_b64 exec, exec 571;CHECK: buffer_load_dword 572;CHECK: buffer_load_dword 573define amdgpu_ps void @test_set_inactive2(i32 inreg %idx0, i32 inreg %idx1) { 574main_body: 575 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 576 %src1.0 = bitcast float %src1 to i32 577 %src1.1 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %src1.0, i32 undef) 578 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 579 %src0.0 = bitcast float %src0 to i32 580 %src0.1 = call i32 @llvm.amdgcn.wqm.i32(i32 %src0.0) 581 %out = add i32 %src0.1, %src1.1 582 %out.0 = bitcast i32 %out to float 583 call void @llvm.amdgcn.struct.buffer.store.f32(float %out.0, <4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 584 ret void 585} 586 587; Check a case of one branch of an if-else requiring WQM, the other requiring 588; exact. 589; 590; Note: In this particular case, the save-and-restore could be avoided if the 591; analysis understood that the two branches of the if-else are mutually 592; exclusive. 593; 594;CHECK-LABEL: {{^}}test_control_flow_0: 595;CHECK-NEXT: ; %main_body 596;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 597;CHECK-NEXT: s_wqm_b64 exec, exec 598;CHECK: %ELSE 599;CHECK: s_and_saveexec_b64 [[SAVED:s\[[0-9]+:[0-9]+\]]], [[ORIG]] 600;CHECK: store 601;CHECK: s_mov_b64 exec, [[SAVED]] 602;CHECK: %IF 603;CHECK: image_sample 604;CHECK: image_sample 605define amdgpu_ps float @test_control_flow_0(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data) { 606main_body: 607 %cmp = icmp eq i32 %z, 0 608 br i1 %cmp, label %IF, label %ELSE 609 610IF: 611 %c.bc = bitcast i32 %c to float 612 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 613 %tex0 = extractelement <4 x float> %tex, i32 0 614 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 615 %data.if = extractelement <4 x float> %dtex, i32 0 616 br label %END 617 618ELSE: 619 call void @llvm.amdgcn.struct.buffer.store.f32(float %data, <4 x i32> undef, i32 %c, i32 0, i32 0, i32 0) 620 br label %END 621 622END: 623 %r = phi float [ %data.if, %IF ], [ %data, %ELSE ] 624 ret float %r 625} 626 627; Reverse branch order compared to the previous test. 628; 629;CHECK-LABEL: {{^}}test_control_flow_1: 630;CHECK-NEXT: ; %main_body 631;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 632;CHECK-NEXT: s_wqm_b64 exec, exec 633;CHECK: %IF 634;CHECK: image_sample 635;CHECK: image_sample 636;CHECK: %Flow 637;CHECK-NEXT: s_or_saveexec_b64 [[SAVED:s\[[0-9]+:[0-9]+\]]], 638;CHECK-NEXT: s_and_b64 exec, exec, [[ORIG]] 639;CHECK-NEXT: s_and_b64 [[SAVED]], exec, [[SAVED]] 640;CHECK-NEXT: s_xor_b64 exec, exec, [[SAVED]] 641;CHECK-NEXT: s_cbranch_execz [[END_BB:BB[0-9]+_[0-9]+]] 642;CHECK-NEXT: ; %bb.{{[0-9]+}}: ; %ELSE 643;CHECK: store_dword 644;CHECK: [[END_BB]]: ; %END 645;CHECK: s_or_b64 exec, exec, 646;CHECK: v_mov_b32_e32 v0 647;CHECK: ; return 648define amdgpu_ps float @test_control_flow_1(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data) { 649main_body: 650 %cmp = icmp eq i32 %z, 0 651 br i1 %cmp, label %ELSE, label %IF 652 653IF: 654 %c.bc = bitcast i32 %c to float 655 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 656 %tex0 = extractelement <4 x float> %tex, i32 0 657 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 658 %data.if = extractelement <4 x float> %dtex, i32 0 659 br label %END 660 661ELSE: 662 call void @llvm.amdgcn.struct.buffer.store.f32(float %data, <4 x i32> undef, i32 %c, i32 0, i32 0, i32 0) 663 br label %END 664 665END: 666 %r = phi float [ %data.if, %IF ], [ %data, %ELSE ] 667 ret float %r 668} 669 670; Check that branch conditions are properly marked as needing WQM... 671; 672;CHECK-LABEL: {{^}}test_control_flow_2: 673;CHECK-NEXT: ; %main_body 674;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 675;CHECK-NEXT: s_wqm_b64 exec, exec 676;CHECK: s_and_b64 exec, exec, [[ORIG]] 677;CHECK: store 678;CHECK: s_wqm_b64 exec, exec 679;CHECK: load 680;CHECK: s_and_b64 exec, exec, [[ORIG]] 681;CHECK: store 682;CHECK: s_wqm_b64 exec, exec 683;CHECK: v_cmp 684define amdgpu_ps <4 x float> @test_control_flow_2(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, <3 x i32> %idx, <2 x float> %data, i32 %coord) { 685main_body: 686 %idx.1 = extractelement <3 x i32> %idx, i32 0 687 %data.1 = extractelement <2 x float> %data, i32 0 688 call void @llvm.amdgcn.struct.buffer.store.f32(float %data.1, <4 x i32> undef, i32 %idx.1, i32 0, i32 0, i32 0) 689 690 ; The load that determines the branch (and should therefore be WQM) is 691 ; surrounded by stores that require disabled WQM. 692 %idx.2 = extractelement <3 x i32> %idx, i32 1 693 %z = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx.2, i32 0, i32 0, i32 0) 694 695 %idx.3 = extractelement <3 x i32> %idx, i32 2 696 %data.3 = extractelement <2 x float> %data, i32 1 697 call void @llvm.amdgcn.struct.buffer.store.f32(float %data.3, <4 x i32> undef, i32 %idx.3, i32 0, i32 0, i32 0) 698 699 %cc = fcmp ogt float %z, 0.0 700 br i1 %cc, label %IF, label %ELSE 701 702IF: 703 %coord.IF = mul i32 %coord, 3 704 br label %END 705 706ELSE: 707 %coord.ELSE = mul i32 %coord, 4 708 br label %END 709 710END: 711 %coord.END = phi i32 [ %coord.IF, %IF ], [ %coord.ELSE, %ELSE ] 712 %coord.END.bc = bitcast i32 %coord.END to float 713 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord.END.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 714 ret <4 x float> %tex 715} 716 717; ... but only if they really do need it. 718; 719;CHECK-LABEL: {{^}}test_control_flow_3: 720;CHECK-NEXT: ; %main_body 721;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 722;CHECK-NEXT: s_wqm_b64 exec, exec 723;CHECK: image_sample 724;CHECK: s_and_b64 exec, exec, [[ORIG]] 725;CHECK: image_sample 726;CHECK-DAG: v_cmp 727;CHECK-DAG: store 728define amdgpu_ps float @test_control_flow_3(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %idx, float %coord) { 729main_body: 730 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 731 %tex0 = extractelement <4 x float> %tex, i32 0 732 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 733 %dtex.1 = extractelement <4 x float> %dtex, i32 0 734 call void @llvm.amdgcn.struct.buffer.store.f32(float %dtex.1, <4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 735 736 %cc = fcmp ogt float %dtex.1, 0.0 737 br i1 %cc, label %IF, label %ELSE 738 739IF: 740 %tex.IF = fmul float %dtex.1, 3.0 741 br label %END 742 743ELSE: 744 %tex.ELSE = fmul float %dtex.1, 4.0 745 br label %END 746 747END: 748 %tex.END = phi float [ %tex.IF, %IF ], [ %tex.ELSE, %ELSE ] 749 ret float %tex.END 750} 751 752; Another test that failed at some point because of terminator handling. 753; 754;CHECK-LABEL: {{^}}test_control_flow_4: 755;CHECK-NEXT: ; %main_body 756;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 757;CHECK-NEXT: s_wqm_b64 exec, exec 758;CHECK: %IF 759;CHECK: s_and_saveexec_b64 [[SAVE:s\[[0-9]+:[0-9]+\]]], [[ORIG]] 760;CHECK: load 761;CHECK: store 762;CHECK: s_mov_b64 exec, [[SAVE]] 763;CHECK: %END 764;CHECK: image_sample 765;CHECK: image_sample 766define amdgpu_ps <4 x float> @test_control_flow_4(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float %coord, i32 %y, float %z) { 767main_body: 768 %cond = icmp eq i32 %y, 0 769 br i1 %cond, label %IF, label %END 770 771IF: 772 %data = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> undef, i32 0, i32 0, i32 0) 773 call void @llvm.amdgcn.struct.buffer.store.f32(float %data, <4 x i32> undef, i32 1, i32 0, i32 0, i32 0) 774 br label %END 775 776END: 777 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 778 %tex0 = extractelement <4 x float> %tex, i32 0 779 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 780 ret <4 x float> %dtex 781} 782 783; Kill is performed in WQM mode so that uniform kill behaves correctly ... 784; 785;CHECK-LABEL: {{^}}test_kill_0: 786;CHECK-NEXT: ; %main_body 787;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 788;CHECK-NEXT: s_wqm_b64 exec, exec 789;CHECK: s_and_b64 exec, exec, [[ORIG]] 790;CHECK: image_sample 791;CHECK: buffer_store_dword 792;CHECK: s_wqm_b64 exec, exec 793;CHECK: v_cmp_ 794;CHECK: image_sample 795;CHECK: s_and_b64 exec, exec, [[ORIG]] 796;CHECK: image_sample 797;CHECK: buffer_store_dword 798define amdgpu_ps <4 x float> @test_kill_0(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float addrspace(1)* inreg %ptr, <2 x i32> %idx, <2 x float> %data, float %coord, float %coord2, float %z) { 799main_body: 800 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 801 %idx.0 = extractelement <2 x i32> %idx, i32 0 802 %data.0 = extractelement <2 x float> %data, i32 0 803 call void @llvm.amdgcn.struct.buffer.store.f32(float %data.0, <4 x i32> undef, i32 %idx.0, i32 0, i32 0, i32 0) 804 805 %z.cmp = fcmp olt float %z, 0.0 806 call void @llvm.amdgcn.kill(i1 %z.cmp) 807 808 %idx.1 = extractelement <2 x i32> %idx, i32 1 809 %data.1 = extractelement <2 x float> %data, i32 1 810 call void @llvm.amdgcn.struct.buffer.store.f32(float %data.1, <4 x i32> undef, i32 %idx.1, i32 0, i32 0, i32 0) 811 %tex2 = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord2, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 812 %tex2.0 = extractelement <4 x float> %tex2, i32 0 813 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex2.0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 814 %out = fadd <4 x float> %tex, %dtex 815 816 ret <4 x float> %out 817} 818 819; ... but only if WQM is necessary. 820; 821; CHECK-LABEL: {{^}}test_kill_1: 822; CHECK-NEXT: ; %main_body 823; CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 824; CHECK: s_wqm_b64 exec, exec 825; CHECK: image_sample 826; CHECK: s_and_b64 exec, exec, [[ORIG]] 827; CHECK: image_sample 828; CHECK-NOT: wqm 829; CHECK-DAG: buffer_store_dword 830; CHECK-DAG: v_cmp_ 831define amdgpu_ps <4 x float> @test_kill_1(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %idx, float %data, float %coord, float %coord2, float %z) { 832main_body: 833 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 834 %tex0 = extractelement <4 x float> %tex, i32 0 835 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 836 837 call void @llvm.amdgcn.raw.buffer.store.f32(float %data, <4 x i32> undef, i32 0, i32 0, i32 0) 838 839 %z.cmp = fcmp olt float %z, 0.0 840 call void @llvm.amdgcn.kill(i1 %z.cmp) 841 842 ret <4 x float> %dtex 843} 844 845; Check prolog shaders. 846; 847; CHECK-LABEL: {{^}}test_prolog_1: 848; CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 849; CHECK: s_wqm_b64 exec, exec 850; CHECK: v_add_f32_e32 v0, 851; CHECK: s_and_b64 exec, exec, [[ORIG]] 852define amdgpu_ps float @test_prolog_1(float %a, float %b) #5 { 853main_body: 854 %s = fadd float %a, %b 855 ret float %s 856} 857 858; CHECK-LABEL: {{^}}test_loop_vcc: 859; CHECK-NEXT: ; %entry 860; CHECK-NEXT: s_mov_b64 [[LIVE:s\[[0-9]+:[0-9]+\]]], exec 861; CHECK: s_wqm_b64 exec, exec 862; CHECK: s_and_b64 exec, exec, [[LIVE]] 863; CHECK: image_store 864; CHECK: s_wqm_b64 exec, exec 865; CHECK-DAG: v_mov_b32_e32 [[CTR:v[0-9]+]], 0 866; CHECK-DAG: s_mov_b32 [[SEVEN:s[0-9]+]], 0x40e00000 867 868; CHECK: [[LOOPHDR:BB[0-9]+_[0-9]+]]: ; %body 869; CHECK: v_add_f32_e32 [[CTR]], 2.0, [[CTR]] 870; CHECK: [[LOOP:BB[0-9]+_[0-9]+]]: ; %loop 871; CHECK: v_cmp_lt_f32_e32 vcc, [[SEVEN]], [[CTR]] 872; CHECK: s_cbranch_vccz [[LOOPHDR]] 873 874; CHECK: ; %break 875; CHECK: ; return 876define amdgpu_ps <4 x float> @test_loop_vcc(<4 x float> %in) nounwind { 877entry: 878 call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> %in, i32 15, i32 undef, <8 x i32> undef, i32 0, i32 0) 879 br label %loop 880 881loop: 882 %ctr.iv = phi float [ 0.0, %entry ], [ %ctr.next, %body ] 883 %c.iv = phi <4 x float> [ %in, %entry ], [ %c.next, %body ] 884 %cc = fcmp ogt float %ctr.iv, 7.0 885 br i1 %cc, label %break, label %body 886 887body: 888 %c.iv0 = extractelement <4 x float> %c.iv, i32 0 889 %c.next = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.iv0, <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0) #0 890 %ctr.next = fadd float %ctr.iv, 2.0 891 br label %loop 892 893break: 894 ret <4 x float> %c.iv 895} 896 897; Only intrinsic stores need exact execution -- other stores do not have 898; externally visible effects and may require WQM for correctness. 899; 900; CHECK-LABEL: {{^}}test_alloca: 901; CHECK: s_mov_b64 [[LIVE:s\[[0-9]+:[0-9]+\]]], exec 902; CHECK: s_wqm_b64 exec, exec 903 904; CHECK: s_and_b64 exec, exec, [[LIVE]] 905; CHECK: buffer_store_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 906; CHECK: s_wqm_b64 exec, exec 907; CHECK: buffer_store_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:4{{$}} 908; CHECK: s_and_b64 exec, exec, [[LIVE]] 909; CHECK: buffer_store_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 idxen 910; CHECK: s_wqm_b64 exec, exec 911; CHECK: buffer_load_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 offen 912 913; CHECK: s_and_b64 exec, exec, [[LIVE]] 914; CHECK: image_sample 915; CHECK: buffer_store_dwordx4 916define amdgpu_ps void @test_alloca(float %data, i32 %a, i32 %idx) nounwind { 917entry: 918 %array = alloca [32 x i32], align 4, addrspace(5) 919 920 call void @llvm.amdgcn.raw.buffer.store.f32(float %data, <4 x i32> undef, i32 0, i32 0, i32 0) 921 922 %s.gep = getelementptr [32 x i32], [32 x i32] addrspace(5)* %array, i32 0, i32 0 923 store volatile i32 %a, i32 addrspace(5)* %s.gep, align 4 924 925 call void @llvm.amdgcn.struct.buffer.store.f32(float %data, <4 x i32> undef, i32 1, i32 0, i32 0, i32 0) 926 927 %c.gep = getelementptr [32 x i32], [32 x i32] addrspace(5)* %array, i32 0, i32 %idx 928 %c = load i32, i32 addrspace(5)* %c.gep, align 4 929 %c.bc = bitcast i32 %c to float 930 %t = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0) #0 931 call void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float> %t, <4 x i32> undef, i32 0, i32 0, i32 0) 932 933 ret void 934} 935 936; Must return to exact at the end of a non-void returning shader, 937; otherwise the EXEC mask exported by the epilog will be wrong. This is true 938; even if the shader has no kills, because a kill could have happened in a 939; previous shader fragment. 940; 941; CHECK-LABEL: {{^}}test_nonvoid_return: 942; CHECK: s_mov_b64 [[LIVE:s\[[0-9]+:[0-9]+\]]], exec 943; CHECK: s_wqm_b64 exec, exec 944; 945; CHECK: s_and_b64 exec, exec, [[LIVE]] 946; CHECK-NOT: exec 947define amdgpu_ps <4 x float> @test_nonvoid_return() nounwind { 948 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float undef, <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0) #0 949 %tex0 = extractelement <4 x float> %tex, i32 0 950 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0) #0 951 ret <4 x float> %dtex 952} 953 954; CHECK-LABEL: {{^}}test_nonvoid_return_unreachable: 955; CHECK: s_mov_b64 [[LIVE:s\[[0-9]+:[0-9]+\]]], exec 956; CHECK: s_wqm_b64 exec, exec 957; 958; CHECK: s_and_b64 exec, exec, [[LIVE]] 959; CHECK-NOT: exec 960define amdgpu_ps <4 x float> @test_nonvoid_return_unreachable(i32 inreg %c) nounwind { 961entry: 962 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float undef, <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0) #0 963 %tex0 = extractelement <4 x float> %tex, i32 0 964 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0) #0 965 %cc = icmp sgt i32 %c, 0 966 br i1 %cc, label %if, label %else 967 968if: 969 store volatile <4 x float> %dtex, <4 x float> addrspace(1)* undef 970 unreachable 971 972else: 973 ret <4 x float> %dtex 974} 975 976; Test awareness that s_wqm_b64 clobbers SCC. 977; 978; CHECK-LABEL: {{^}}test_scc: 979; CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 980; CHECK: s_wqm_b64 exec, exec 981; CHECK: s_cmp_ 982; CHECK-NEXT: s_cbranch_scc 983; CHECK: ; %else 984; CHECK: image_sample 985; CHECK: ; %if 986; CHECK: image_sample 987; CHECK: ; %end 988; CHECK: s_and_b64 exec, exec, [[ORIG]] 989define amdgpu_ps <4 x float> @test_scc(i32 inreg %sel, i32 %idx) #1 { 990main_body: 991 %cc = icmp sgt i32 %sel, 0 992 br i1 %cc, label %if, label %else 993 994if: 995 %r.if = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float 0.0, <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0) #0 996 br label %end 997 998else: 999 %r.else = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32 15, float 0.0, float bitcast (i32 1 to float), <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0) #0 1000 br label %end 1001 1002end: 1003 %r = phi <4 x float> [ %r.if, %if ], [ %r.else, %else ] 1004 call void @llvm.amdgcn.struct.buffer.store.f32(float 1.0, <4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 1005 ret <4 x float> %r 1006} 1007 1008; Check a case of a block being entirely WQM except for a bit of WWM. 1009; There was a bug where it forgot to enter and leave WWM. 1010; 1011;CHECK-LABEL: {{^}}test_wwm_within_wqm: 1012;CHECK: %IF 1013;CHECK: s_or_saveexec_b64 {{.*}}, -1 1014;CHECK: ds_swizzle 1015; 1016define amdgpu_ps float @test_wwm_within_wqm(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data) { 1017main_body: 1018 %c.bc = bitcast i32 %c to float 1019 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1020 %tex0 = extractelement <4 x float> %tex, i32 0 1021 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1022 %cmp = icmp eq i32 %z, 0 1023 br i1 %cmp, label %IF, label %ENDIF 1024 1025IF: 1026 %dataf = extractelement <4 x float> %dtex, i32 0 1027 %data1 = fptosi float %dataf to i32 1028 %data2 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %data1, i32 0) 1029 %data3 = call i32 @llvm.amdgcn.ds.swizzle(i32 %data2, i32 2079) 1030 %data4 = call i32 @llvm.amdgcn.wwm.i32(i32 %data3) 1031 %data4f = sitofp i32 %data4 to float 1032 br label %ENDIF 1033 1034ENDIF: 1035 %r = phi float [ %data4f, %IF ], [ 0.0, %main_body ] 1036 ret float %r 1037} 1038 1039; Check that WWM is triggered by the strict_wwm intrinsic. 1040; 1041;CHECK-LABEL: {{^}}test_strict_wwm1: 1042;CHECK: s_or_saveexec_b64 s{{\[[0-9]+:[0-9]+\]}}, -1 1043;CHECK: buffer_load_dword 1044;CHECK: buffer_load_dword 1045;CHECK: v_add_f32_e32 1046define amdgpu_ps float @test_strict_wwm1(i32 inreg %idx0, i32 inreg %idx1) { 1047main_body: 1048 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 1049 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 1050 %out = fadd float %src0, %src1 1051 %out.0 = call float @llvm.amdgcn.strict.wwm.f32(float %out) 1052 ret float %out.0 1053} 1054 1055; Same as above, but with an integer type. 1056; 1057;CHECK-LABEL: {{^}}test_strict_wwm2: 1058;CHECK: s_or_saveexec_b64 s{{\[[0-9]+:[0-9]+\]}}, -1 1059;CHECK: buffer_load_dword 1060;CHECK: buffer_load_dword 1061;CHECK: v_add_{{[iu]}}32_e32 1062define amdgpu_ps float @test_strict_wwm2(i32 inreg %idx0, i32 inreg %idx1) { 1063main_body: 1064 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 1065 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 1066 %src0.0 = bitcast float %src0 to i32 1067 %src1.0 = bitcast float %src1 to i32 1068 %out = add i32 %src0.0, %src1.0 1069 %out.0 = call i32 @llvm.amdgcn.strict.wwm.i32(i32 %out) 1070 %out.1 = bitcast i32 %out.0 to float 1071 ret float %out.1 1072} 1073 1074; Check that we don't leave WWM on for computations that don't require WWM, 1075; since that will lead clobbering things that aren't supposed to be clobbered 1076; in cases like this. 1077; We enforce this by checking that v_add gets emitted in the same block as 1078; WWM computations. 1079; 1080;CHECK-LABEL: {{^}}test_strict_wwm3: 1081;CHECK: %if 1082;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1 1083;CHECK: buffer_load_dword 1084;CHECK: v_add_f32_e32 1085;CHECK: s_mov_b64 exec, [[ORIG]] 1086;CHECK: v_add_f32_e32 1087;CHECK: %endif 1088define amdgpu_ps float @test_strict_wwm3(i32 inreg %idx) { 1089main_body: 1090 ; use mbcnt to make sure the branch is divergent 1091 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 1092 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 1093 %cc = icmp uge i32 %hi, 32 1094 br i1 %cc, label %endif, label %if 1095 1096if: 1097 %src = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 1098 %out = fadd float %src, %src 1099 %out.0 = call float @llvm.amdgcn.strict.wwm.f32(float %out) 1100 %out.1 = fadd float %src, %out.0 1101 br label %endif 1102 1103endif: 1104 %out.2 = phi float [ %out.1, %if ], [ 0.0, %main_body ] 1105 ret float %out.2 1106} 1107 1108; Check that WWM writes aren't coalesced with non-WWM writes, since the WWM 1109; write could clobber disabled channels in the non-WWM one. 1110; We enforce this by checking that v_mov gets emitted in the same block as 1111; WWM computations. 1112; 1113;CHECK-LABEL: {{^}}test_strict_wwm4: 1114;CHECK: %if 1115;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1 1116;CHECK: buffer_load_dword 1117;CHECK: v_add_f32_e32 1118;CHECK: s_mov_b64 exec, [[ORIG]] 1119;CHECK-NEXT: v_mov_b32_e32 1120;CHECK: %endif 1121define amdgpu_ps float @test_strict_wwm4(i32 inreg %idx) { 1122main_body: 1123 ; use mbcnt to make sure the branch is divergent 1124 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 1125 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 1126 %cc = icmp uge i32 %hi, 32 1127 br i1 %cc, label %endif, label %if 1128 1129if: 1130 %src = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 1131 %out = fadd float %src, %src 1132 %out.0 = call float @llvm.amdgcn.strict.wwm.f32(float %out) 1133 br label %endif 1134 1135endif: 1136 %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ] 1137 ret float %out.1 1138} 1139 1140; Make sure the transition from Exact to WWM then WQM works properly. 1141; 1142;CHECK-LABEL: {{^}}test_strict_wwm5: 1143;CHECK: buffer_load_dword 1144;CHECK: buffer_store_dword 1145;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1 1146;CHECK: buffer_load_dword 1147;CHECK: v_add_f32_e32 1148;CHECK: s_mov_b64 exec, [[ORIG]] 1149;CHECK: s_wqm_b64 exec, exec 1150define amdgpu_ps float @test_strict_wwm5(i32 inreg %idx0, i32 inreg %idx1) { 1151main_body: 1152 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 1153 call void @llvm.amdgcn.struct.buffer.store.f32(float %src0, <4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0) 1154 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0) 1155 %temp = fadd float %src1, %src1 1156 %temp.0 = call float @llvm.amdgcn.strict.wwm.f32(float %temp) 1157 %out = fadd float %temp.0, %temp.0 1158 %out.0 = call float @llvm.amdgcn.wqm.f32(float %out) 1159 ret float %out.0 1160} 1161 1162; Check that WWM is turned on correctly across basic block boundaries. 1163; if..then..endif version 1164; 1165;CHECK-LABEL: {{^}}test_strict_wwm6_then: 1166;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1 1167;SI-CHECK: buffer_load_dword 1168;VI-CHECK: flat_load_dword 1169;CHECK: s_mov_b64 exec, [[ORIG]] 1170;CHECK: %if 1171;CHECK: s_or_saveexec_b64 [[ORIG2:s\[[0-9]+:[0-9]+\]]], -1 1172;SI-CHECK: buffer_load_dword 1173;VI-CHECK: flat_load_dword 1174;CHECK: v_add_f32_e32 1175;CHECK: s_mov_b64 exec, [[ORIG2]] 1176;CHECK: %endif 1177define amdgpu_ps float @test_strict_wwm6_then() { 1178main_body: 1179 %src0 = load volatile float, float addrspace(1)* undef 1180 ; use mbcnt to make sure the branch is divergent 1181 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 1182 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 1183 %cc = icmp uge i32 %hi, 32 1184 br i1 %cc, label %endif, label %if 1185 1186if: 1187 %src1 = load volatile float, float addrspace(1)* undef 1188 %out = fadd float %src0, %src1 1189 %out.0 = call float @llvm.amdgcn.strict.wwm.f32(float %out) 1190 br label %endif 1191 1192endif: 1193 %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ] 1194 ret float %out.1 1195} 1196 1197; Check that WWM is turned on correctly across basic block boundaries. 1198; loop version 1199; 1200;CHECK-LABEL: {{^}}test_strict_wwm6_loop: 1201;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1 1202;SI-CHECK: buffer_load_dword 1203;VI-CHECK: flat_load_dword 1204;CHECK: s_mov_b64 exec, [[ORIG]] 1205;CHECK: %loop 1206;CHECK: s_or_saveexec_b64 [[ORIG2:s\[[0-9]+:[0-9]+\]]], -1 1207;SI-CHECK: buffer_load_dword 1208;VI-CHECK: flat_load_dword 1209;CHECK: s_mov_b64 exec, [[ORIG2]] 1210;CHECK: %endloop 1211define amdgpu_ps float @test_strict_wwm6_loop() { 1212main_body: 1213 %src0 = load volatile float, float addrspace(1)* undef 1214 ; use mbcnt to make sure the branch is divergent 1215 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 1216 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 1217 br label %loop 1218 1219loop: 1220 %counter = phi i32 [ %lo, %main_body ], [ %counter.1, %loop ] 1221 %src1 = load volatile float, float addrspace(1)* undef 1222 %out = fadd float %src0, %src1 1223 %out.0 = call float @llvm.amdgcn.strict.wwm.f32(float %out) 1224 %counter.1 = sub i32 %counter, 1 1225 %cc = icmp ne i32 %counter.1, 0 1226 br i1 %cc, label %loop, label %endloop 1227 1228endloop: 1229 ret float %out.0 1230} 1231 1232; Check that @llvm.amdgcn.set.inactive disables WWM. 1233; 1234;CHECK-LABEL: {{^}}test_strict_wwm_set_inactive1: 1235;CHECK: buffer_load_dword 1236;CHECK: s_not_b64 exec, exec 1237;CHECK: v_mov_b32_e32 1238;CHECK: s_not_b64 exec, exec 1239;CHECK: s_or_saveexec_b64 s{{\[[0-9]+:[0-9]+\]}}, -1 1240;CHECK: v_add_{{[iu]}}32_e32 1241define amdgpu_ps void @test_strict_wwm_set_inactive1(i32 inreg %idx) { 1242main_body: 1243 %src = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 1244 %src.0 = bitcast float %src to i32 1245 %src.1 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %src.0, i32 0) 1246 %out = add i32 %src.1, %src.1 1247 %out.0 = call i32 @llvm.amdgcn.strict.wwm.i32(i32 %out) 1248 %out.1 = bitcast i32 %out.0 to float 1249 call void @llvm.amdgcn.struct.buffer.store.f32(float %out.1, <4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0) 1250 ret void 1251} 1252 1253; Check a case of a block being entirely WQM except for a bit of WWM. 1254; There was a bug where it forgot to enter and leave WWM. 1255; 1256;CHECK-LABEL: {{^}}test_strict_wwm_within_wqm: 1257;CHECK: %IF 1258;CHECK: s_or_saveexec_b64 {{.*}}, -1 1259;CHECK: ds_swizzle 1260; 1261define amdgpu_ps float @test_strict_wwm_within_wqm(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data) { 1262main_body: 1263 %c.bc = bitcast i32 %c to float 1264 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1265 %tex0 = extractelement <4 x float> %tex, i32 0 1266 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1267 %cmp = icmp eq i32 %z, 0 1268 br i1 %cmp, label %IF, label %ENDIF 1269 1270IF: 1271 %dataf = extractelement <4 x float> %dtex, i32 0 1272 %data1 = fptosi float %dataf to i32 1273 %data2 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %data1, i32 0) 1274 %data3 = call i32 @llvm.amdgcn.ds.swizzle(i32 %data2, i32 2079) 1275 %data4 = call i32 @llvm.amdgcn.strict.wwm.i32(i32 %data3) 1276 %data4f = sitofp i32 %data4 to float 1277 br label %ENDIF 1278 1279ENDIF: 1280 %r = phi float [ %data4f, %IF ], [ 0.0, %main_body ] 1281 ret float %r 1282} 1283 1284; Check a case of a block being entirely WQM except for a bit of STRICT WQM. 1285; 1286;CHECK-LABEL: {{^}}test_strict_wqm_within_wqm: 1287;CHECK: %IF 1288;CHECK: s_mov_b64 s{{\[[0-9]+:[0-9]+\]}}, exec 1289;CHECK: s_wqm_b64 exec, exec 1290;CHECK: ds_swizzle 1291; 1292define amdgpu_ps float @test_strict_wqm_within_wqm(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data) { 1293main_body: 1294 %c.bc = bitcast i32 %c to float 1295 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1296 %tex0 = extractelement <4 x float> %tex, i32 0 1297 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0 1298 %cmp = icmp eq i32 %z, 0 1299 br i1 %cmp, label %IF, label %ENDIF 1300 1301IF: 1302 %dataf = extractelement <4 x float> %dtex, i32 0 1303 %data1 = fptosi float %dataf to i32 1304 %data2 = call i32 @llvm.amdgcn.ds.swizzle(i32 %data1, i32 2079) 1305 %data3 = call i32 @llvm.amdgcn.strict.wqm.i32(i32 %data2) 1306 %data3f = sitofp i32 %data3 to float 1307 br label %ENDIF 1308 1309ENDIF: 1310 %r = phi float [ %data3f, %IF ], [ 0.0, %main_body ] 1311 ret float %r 1312} 1313 1314;CHECK-LABEL: {{^}}test_strict_wqm_strict_wwm_wqm: 1315;CHECK: buffer_store_dword 1316 1317;CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 1318;CHECK: s_wqm_b64 exec, exec 1319;CHECK: buffer_load_dword 1320;CHECK: s_mov_b64 exec, [[ORIG]] 1321 1322;CHECK: s_or_saveexec_b64 [[ORIG2:s\[[0-9]+:[0-9]+\]]], -1 1323;CHECK: buffer_load_dword 1324;CHECK: s_mov_b64 exec, [[ORIG2]] 1325 1326;CHECK: s_mov_b64 [[ORIG3:s\[[0-9]+:[0-9]+\]]], exec 1327;CHECK: s_wqm_b64 exec, exec 1328;CHECK: v_add 1329;CHECK: s_mov_b64 exec, [[ORIG3]] 1330 1331;TODO: StrictWQM -> WQM transition could be improved. WQM could use the exec from the previous state instead of calling s_wqm again. 1332;CHECK: s_wqm_b64 exec, exec 1333;CHECK: image_sample 1334 1335define amdgpu_ps float @test_strict_wqm_strict_wwm_wqm(i32 inreg %idx0, i32 inreg %idx1, <4 x i32> inreg %res, <4 x i32> inreg %res2, float %inp, <8 x i32> inreg %res3) { 1336main_body: 1337 call void @llvm.amdgcn.struct.buffer.store.f32(float %inp, <4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0) 1338 %reload = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0) 1339 %temp = fadd float %reload, %reload 1340 %temp2 = call float @llvm.amdgcn.strict.wqm.f32(float %temp) 1341 %temp3 = fadd float %temp2, %temp2 1342 %reload_wwm = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res2, i32 %idx0, i32 0, i32 0, i32 0) 1343 %temp4 = call float @llvm.amdgcn.strict.wwm.f32(float %reload_wwm) 1344 %temp5 = fadd float %temp3, %temp4 1345 %tex = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %temp5, <8 x i32> %res3, <4 x i32> %res, i1 false, i32 0, i32 0) 1346 call void @llvm.amdgcn.struct.buffer.store.f32(float %tex, <4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0) 1347 %out = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0) 1348 ret float %out 1349} 1350 1351;CHECK-LABEL: {{^}}test_strict_wwm_strict_wqm_wqm: 1352;CHECK: buffer_store_dword 1353 1354;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1 1355;CHECK: buffer_load_dword 1356;CHECK: s_mov_b64 exec, [[ORIG]] 1357 1358;CHECK: s_mov_b64 [[ORIG2:s\[[0-9]+:[0-9]+\]]], exec 1359;CHECK: s_wqm_b64 exec, exec 1360;CHECK: buffer_load_dword 1361;CHECK: s_mov_b64 exec, [[ORIG2]] 1362 1363;CHECK: s_or_saveexec_b64 [[ORIG3:s\[[0-9]+:[0-9]+\]]], -1 1364;CHECK: v_add 1365;CHECK: s_mov_b64 exec, [[ORIG3]] 1366 1367;CHECK: s_wqm_b64 exec, exec 1368;CHECK: image_sample 1369define amdgpu_ps float @test_strict_wwm_strict_wqm_wqm(i32 inreg %idx0, i32 inreg %idx1, <4 x i32> inreg %res, float %inp, <8 x i32> inreg %res2) { 1370main_body: 1371 call void @llvm.amdgcn.struct.buffer.store.f32(float %inp, <4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0) 1372 %reload = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0) 1373 %temp = fadd float %reload, %reload 1374 %temp2 = call float @llvm.amdgcn.strict.wwm.f32(float %temp) 1375 %temp3 = fadd float %temp2, %temp2 1376 %reload_wwm = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0) 1377 %temp4 = call float @llvm.amdgcn.strict.wqm.f32(float %reload_wwm) 1378 %temp5 = fadd float %temp3, %temp4 1379 %tex = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %temp5, <8 x i32> %res2, <4 x i32> %res, i1 false, i32 0, i32 0) 1380 call void @llvm.amdgcn.struct.buffer.store.f32(float %tex, <4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0) 1381 %out = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0) 1382 ret float %out 1383} 1384 1385;CHECK-LABEL: {{^}}test_wqm_strict_wqm_wqm: 1386;CHECK: buffer_store_dword 1387 1388;CHECK: s_wqm_b64 exec, exec 1389 1390;TODO: WQM -> StrictWQM transition could be improved. StrictWQM could use the exec from the previous state instead of calling s_wqm again. 1391;CHECK: s_mov_b64 [[ORIG2:s\[[0-9]+:[0-9]+\]]], exec 1392;CHECK: s_wqm_b64 exec, exec 1393;CHECK: buffer_load_dword 1394;CHECK: s_mov_b64 exec, [[ORIG2]] 1395 1396;CHECK: image_sample 1397 1398define amdgpu_ps float @test_wqm_strict_wqm_wqm(i32 inreg %idx0, i32 inreg %idx1, <4 x i32> inreg %res, float %inp, <8 x i32> inreg %res2) { 1399main_body: 1400 call void @llvm.amdgcn.struct.buffer.store.f32(float %inp, <4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0) 1401 %reload = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0) 1402 %temp = fadd float %reload, %reload 1403 %tex = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %temp, <8 x i32> %res2, <4 x i32> %res, i1 false, i32 0, i32 0) 1404 %temp2 = fadd float %tex, %tex 1405 %reload_wwm = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0) 1406 %temp3 = call float @llvm.amdgcn.strict.wqm.f32(float %reload_wwm) 1407 %temp4 = fadd float %temp2, %temp3 1408 %tex2 = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %temp4, <8 x i32> %res2, <4 x i32> %res, i1 false, i32 0, i32 0) 1409 call void @llvm.amdgcn.struct.buffer.store.f32(float %tex2, <4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0) 1410 %out = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0) 1411 ret float %out 1412} 1413 1414declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #1 1415declare void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float>, i32, i32, <8 x i32>, i32, i32) #1 1416 1417declare void @llvm.amdgcn.struct.buffer.store.f32(float, <4 x i32>, i32, i32, i32, i32 immarg) #2 1418declare void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32, i32 immarg) #2 1419declare void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32 immarg) #2 1420declare void @llvm.amdgcn.raw.buffer.store.f32(float, <4 x i32>, i32, i32, i32 immarg) #2 1421declare float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32>, i32, i32, i32) #3 1422declare float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32>, i32, i32, i32, i32) #3 1423 1424declare <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32, i32, <8 x i32>, i32, i32) #3 1425declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #3 1426declare <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #3 1427declare float @llvm.amdgcn.image.sample.1d.f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #3 1428declare void @llvm.amdgcn.kill(i1) #1 1429declare float @llvm.amdgcn.wqm.f32(float) #3 1430declare i32 @llvm.amdgcn.wqm.i32(i32) #3 1431declare float @llvm.amdgcn.strict.wwm.f32(float) #3 1432declare i32 @llvm.amdgcn.strict.wwm.i32(i32) #3 1433declare float @llvm.amdgcn.wwm.f32(float) #3 1434declare i32 @llvm.amdgcn.wwm.i32(i32) #3 1435declare float @llvm.amdgcn.strict.wqm.f32(float) #3 1436declare i32 @llvm.amdgcn.strict.wqm.i32(i32) #3 1437declare i32 @llvm.amdgcn.set.inactive.i32(i32, i32) #4 1438declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #3 1439declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32) #3 1440declare <2 x half> @llvm.amdgcn.cvt.pkrtz(float, float) #3 1441declare void @llvm.amdgcn.exp.compr.v2f16(i32, i32, <2 x half>, <2 x half>, i1, i1) #1 1442declare float @llvm.amdgcn.interp.p1(float, i32, i32, i32) #2 1443declare float @llvm.amdgcn.interp.p2(float, float, i32, i32, i32) #2 1444declare i32 @llvm.amdgcn.ds.swizzle(i32, i32) 1445 1446attributes #1 = { nounwind } 1447attributes #2 = { nounwind readonly } 1448attributes #3 = { nounwind readnone } 1449attributes #4 = { nounwind readnone convergent } 1450attributes #5 = { "amdgpu-ps-wqm-outputs" } 1451attributes #6 = { nounwind "InitialPSInputAddr"="2" } 1452