1; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=CHECK -check-prefix=SI %s 2; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=CHECK -check-prefix=VI %s 3 4; Check that WQM isn't triggered by image load/store intrinsics. 5; 6;CHECK-LABEL: {{^}}test1: 7;CHECK-NOT: s_wqm 8define amdgpu_ps <4 x float> @test1(<8 x i32> inreg %rsrc, <4 x i32> %c) { 9main_body: 10 %tex = call <4 x float> @llvm.amdgcn.image.load.v4f32.v4i32.v8i32(<4 x i32> %c, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) 11 call void @llvm.amdgcn.image.store.v4f32.v4i32.v8i32(<4 x float> %tex, <4 x i32> %c, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) 12 ret <4 x float> %tex 13} 14 15; Check that WQM is triggered by code calculating inputs to image samples and is disabled as soon as possible 16; 17;CHECK-LABEL: {{^}}test2: 18;CHECK-NEXT: ; %main_body 19;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 20;CHECK-NEXT: s_wqm_b64 exec, exec 21;CHECK: interp 22;CHECK: s_and_b64 exec, exec, [[ORIG]] 23;CHECK-NOT: interp 24;CHECK: image_sample 25;CHECK-NOT: exec 26;CHECK: .size test2 27define amdgpu_ps <4 x float> @test2(i32 inreg, i32 inreg, i32 inreg, i32 inreg %m0, <8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, <2 x float> %pos) #6 { 28main_body: 29 %inst23 = extractelement <2 x float> %pos, i32 0 30 %inst24 = extractelement <2 x float> %pos, i32 1 31 %inst25 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 0, i32 0, i32 %m0) 32 %inst26 = tail call float @llvm.amdgcn.interp.p2(float %inst25, float %inst24, i32 0, i32 0, i32 %m0) 33 %inst27 = insertelement <2 x float> undef, float %inst26, i32 0 34 %inst28 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 1, i32 0, i32 %m0) 35 %inst29 = tail call float @llvm.amdgcn.interp.p2(float %inst28, float %inst24, i32 1, i32 0, i32 %m0) 36 %inst30 = insertelement <2 x float> %inst27, float %inst29, i32 1 37 %tex = call <4 x float> @llvm.amdgcn.image.sample.v4f32.v2f32.v8i32(<2 x float> %inst30, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0 38 ret <4 x float> %tex 39} 40 41; ... but disabled for stores (and, in this simple case, not re-enabled) ... 42; 43;CHECK-LABEL: {{^}}test3: 44;CHECK-NEXT: ; %main_body 45;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 46;CHECK-NEXT: s_wqm_b64 exec, exec 47;CHECK: s_and_b64 exec, exec, [[ORIG]] 48;CHECK: image_sample 49;CHECK: store 50;CHECK-NOT: exec 51;CHECK: .size test3 52define amdgpu_ps <4 x float> @test3(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, <4 x float> %c) { 53main_body: 54 %tex = call <4 x float> @llvm.amdgcn.image.sample.v4f32.v4f32.v8i32(<4 x float> %c, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0 55 %tex.1 = bitcast <4 x float> %tex to <4 x i32> 56 %tex.2 = extractelement <4 x i32> %tex.1, i32 0 57 58 call void @llvm.amdgcn.buffer.store.v4f32(<4 x float> %tex, <4 x i32> undef, i32 %tex.2, i32 0, i1 0, i1 0) 59 60 ret <4 x float> %tex 61} 62 63; ... and disabled for export. 64; 65;CHECK-LABEL: {{^}}test3x: 66;CHECK-NEXT: ; %main_body 67;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 68;CHECK-NEXT: s_wqm_b64 exec, exec 69;CHECK: s_and_b64 exec, exec, [[ORIG]] 70;CHECK: image_sample 71;CHECK: exp 72;CHECK-NOT: exec 73;CHECK: .size test3x 74define amdgpu_ps void @test3x(i32 inreg, i32 inreg, i32 inreg, i32 inreg %m0, <8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, <2 x float> %pos) #6 { 75main_body: 76 %inst23 = extractelement <2 x float> %pos, i32 0 77 %inst24 = extractelement <2 x float> %pos, i32 1 78 %inst25 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 0, i32 0, i32 %m0) 79 %inst26 = tail call float @llvm.amdgcn.interp.p2(float %inst25, float %inst24, i32 0, i32 0, i32 %m0) 80 %inst27 = insertelement <2 x float> undef, float %inst26, i32 0 81 %inst28 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 1, i32 0, i32 %m0) 82 %inst29 = tail call float @llvm.amdgcn.interp.p2(float %inst28, float %inst24, i32 1, i32 0, i32 %m0) 83 %inst30 = insertelement <2 x float> %inst27, float %inst29, i32 1 84 %tex = call <4 x float> @llvm.amdgcn.image.sample.v4f32.v2f32.v8i32(<2 x float> %inst30, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0 85 %tex.0 = extractelement <4 x float> %tex, i32 0 86 %tex.1 = extractelement <4 x float> %tex, i32 1 87 %tex.2 = extractelement <4 x float> %tex, i32 2 88 %tex.3 = extractelement <4 x float> %tex, i32 3 89 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %tex.0, float %tex.1, float %tex.2, float %tex.3, i1 true, i1 true) 90 ret void 91} 92 93; Check that WQM is re-enabled when required. 94; 95;CHECK-LABEL: {{^}}test4: 96;CHECK-NEXT: ; %main_body 97;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 98;CHECK-NEXT: s_wqm_b64 exec, exec 99;CHECK: v_mul_lo_i32 [[MUL:v[0-9]+]], v0, v1 100;CHECK: s_and_b64 exec, exec, [[ORIG]] 101;CHECK: store 102;CHECK: s_wqm_b64 exec, exec 103;CHECK: image_sample 104;CHECK: image_sample 105define amdgpu_ps <4 x float> @test4(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float addrspace(1)* inreg %ptr, i32 %c, i32 %d, float %data) { 106main_body: 107 %c.1 = mul i32 %c, %d 108 109 call void @llvm.amdgcn.buffer.store.v4f32(<4 x float> undef, <4 x i32> undef, i32 %c.1, i32 0, i1 0, i1 0) 110 %c.1.bc = bitcast i32 %c.1 to float 111 %tex = call <4 x float> @llvm.amdgcn.image.sample.v4f32.f32.v8i32(float %c.1.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0 112 %dtex = call <4 x float> @llvm.amdgcn.image.sample.v4f32.v4f32.v8i32(<4 x float> %tex, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0 113 ret <4 x float> %dtex 114} 115 116; Check that WQM is triggered by the wqm intrinsic. 117; 118;CHECK-LABEL: {{^}}test5: 119;CHECK: s_wqm_b64 exec, exec 120;CHECK: buffer_load_dword 121;CHECK: buffer_load_dword 122;CHECK: v_add_f32_e32 123define amdgpu_ps float @test5(i32 inreg %idx0, i32 inreg %idx1) { 124main_body: 125 %src0 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0) 126 %src1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i1 0, i1 0) 127 %out = fadd float %src0, %src1 128 %out.0 = call float @llvm.amdgcn.wqm.f32(float %out) 129 ret float %out.0 130} 131 132; Check that the wqm intrinsic works correctly for integers. 133; 134;CHECK-LABEL: {{^}}test6: 135;CHECK: s_wqm_b64 exec, exec 136;CHECK: buffer_load_dword 137;CHECK: buffer_load_dword 138;CHECK: v_add_f32_e32 139define amdgpu_ps float @test6(i32 inreg %idx0, i32 inreg %idx1) { 140main_body: 141 %src0 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0) 142 %src1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i1 0, i1 0) 143 %out = fadd float %src0, %src1 144 %out.0 = bitcast float %out to i32 145 %out.1 = call i32 @llvm.amdgcn.wqm.i32(i32 %out.0) 146 %out.2 = bitcast i32 %out.1 to float 147 ret float %out.2 148} 149 150; Check that WWM is triggered by the wwm intrinsic. 151; 152;CHECK-LABEL: {{^}}test_wwm1: 153;CHECK: s_or_saveexec_b64 s{{\[[0-9]+:[0-9]+\]}}, -1 154;CHECK: buffer_load_dword 155;CHECK: buffer_load_dword 156;CHECK: v_add_f32_e32 157define amdgpu_ps float @test_wwm1(i32 inreg %idx0, i32 inreg %idx1) { 158main_body: 159 %src0 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0) 160 %src1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i1 0, i1 0) 161 %out = fadd float %src0, %src1 162 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out) 163 ret float %out.0 164} 165 166; Same as above, but with an integer type. 167; 168;CHECK-LABEL: {{^}}test_wwm2: 169;CHECK: s_or_saveexec_b64 s{{\[[0-9]+:[0-9]+\]}}, -1 170;CHECK: buffer_load_dword 171;CHECK: buffer_load_dword 172;CHECK: v_add_i32_e32 173define amdgpu_ps float @test_wwm2(i32 inreg %idx0, i32 inreg %idx1) { 174main_body: 175 %src0 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0) 176 %src1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i1 0, i1 0) 177 %src0.0 = bitcast float %src0 to i32 178 %src1.0 = bitcast float %src1 to i32 179 %out = add i32 %src0.0, %src1.0 180 %out.0 = call i32 @llvm.amdgcn.wwm.i32(i32 %out) 181 %out.1 = bitcast i32 %out.0 to float 182 ret float %out.1 183} 184 185; Check that we don't leave WWM on for computations that don't require WWM, 186; since that will lead clobbering things that aren't supposed to be clobbered 187; in cases like this. 188; 189;CHECK-LABEL: {{^}}test_wwm3: 190;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1 191;CHECK: buffer_load_dword 192;CHECK: v_add_f32_e32 193;CHECK: s_mov_b64 exec, [[ORIG]] 194;CHECK: v_add_f32_e32 195define amdgpu_ps float @test_wwm3(i32 inreg %idx) { 196main_body: 197 ; use mbcnt to make sure the branch is divergent 198 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 199 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 200 %cc = icmp uge i32 %hi, 32 201 br i1 %cc, label %endif, label %if 202 203if: 204 %src = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i1 0, i1 0) 205 %out = fadd float %src, %src 206 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out) 207 %out.1 = fadd float %src, %out.0 208 br label %endif 209 210endif: 211 %out.2 = phi float [ %out.1, %if ], [ 0.0, %main_body ] 212 ret float %out.2 213} 214 215; Check that WWM writes aren't coalesced with non-WWM writes, since the WWM 216; write could clobber disabled channels in the non-WWM one. 217; 218;CHECK-LABEL: {{^}}test_wwm4: 219;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1 220;CHECK: buffer_load_dword 221;CHECK: v_add_f32_e32 222;CHECK: s_mov_b64 exec, [[ORIG]] 223;CHECK-NEXT: v_mov_b32_e32 224define amdgpu_ps float @test_wwm4(i32 inreg %idx) { 225main_body: 226 ; use mbcnt to make sure the branch is divergent 227 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 228 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 229 %cc = icmp uge i32 %hi, 32 230 br i1 %cc, label %endif, label %if 231 232if: 233 %src = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i1 0, i1 0) 234 %out = fadd float %src, %src 235 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out) 236 br label %endif 237 238endif: 239 %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ] 240 ret float %out.1 241} 242 243; Make sure the transition from Exact to WWM then WQM works properly. 244; 245;CHECK-LABEL: {{^}}test_wwm5: 246;CHECK: buffer_load_dword 247;CHECK: buffer_store_dword 248;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1 249;CHECK: buffer_load_dword 250;CHECK: v_add_f32_e32 251;CHECK: s_mov_b64 exec, [[ORIG]] 252;CHECK: s_wqm_b64 exec, exec 253define amdgpu_ps float @test_wwm5(i32 inreg %idx0, i32 inreg %idx1) { 254main_body: 255 %src0 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0) 256 call void @llvm.amdgcn.buffer.store.f32(float %src0, <4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0) 257 %src1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i1 0, i1 0) 258 %temp = fadd float %src1, %src1 259 %temp.0 = call float @llvm.amdgcn.wwm.f32(float %temp) 260 %out = fadd float %temp.0, %temp.0 261 %out.0 = call float @llvm.amdgcn.wqm.f32(float %out) 262 ret float %out.0 263} 264 265; Check that WWM is turned on correctly across basic block boundaries. 266; 267;CHECK-LABEL: {{^}}test_wwm6: 268;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1 269;SI-CHECK: buffer_load_dword 270;VI-CHECK: flat_load_dword 271;CHECK: s_mov_b64 exec, [[ORIG]] 272;CHECK: %if 273;CHECK: s_or_saveexec_b64 [[ORIG2:s\[[0-9]+:[0-9]+\]]], -1 274;SI-CHECK: buffer_load_dword 275;VI-CHECK: flat_load_dword 276;CHECK: v_add_f32_e32 277;CHECK: s_mov_b64 exec, [[ORIG2]] 278define amdgpu_ps float @test_wwm6() { 279main_body: 280 %src0 = load volatile float, float addrspace(1)* undef 281 ; use mbcnt to make sure the branch is divergent 282 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) 283 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo) 284 %cc = icmp uge i32 %hi, 32 285 br i1 %cc, label %endif, label %if 286 287if: 288 %src1 = load volatile float, float addrspace(1)* undef 289 %out = fadd float %src0, %src1 290 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out) 291 br label %endif 292 293endif: 294 %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ] 295 ret float %out.1 296} 297 298; Check that @llvm.amdgcn.set.inactive disables WWM. 299; 300;CHECK-LABEL: {{^}}test_set_inactive1: 301;CHECK: buffer_load_dword 302;CHECK: s_not_b64 exec, exec 303;CHECK: v_mov_b32_e32 304;CHECK: s_not_b64 exec, exec 305;CHECK: s_or_saveexec_b64 s{{\[[0-9]+:[0-9]+\]}}, -1 306;CHECK: v_add_i32_e32 307define amdgpu_ps void @test_set_inactive1(i32 inreg %idx) { 308main_body: 309 %src = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i1 0, i1 0) 310 %src.0 = bitcast float %src to i32 311 %src.1 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %src.0, i32 0) 312 %out = add i32 %src.1, %src.1 313 %out.0 = call i32 @llvm.amdgcn.wwm.i32(i32 %out) 314 %out.1 = bitcast i32 %out.0 to float 315 call void @llvm.amdgcn.buffer.store.f32(float %out.1, <4 x i32> undef, i32 %idx, i32 0, i1 0, i1 0) 316 ret void 317} 318 319; Check that enabling WQM anywhere enables WQM for the set.inactive source. 320; 321;CHECK-LABEL: {{^}}test_set_inactive2: 322;CHECK: s_wqm_b64 exec, exec 323;CHECK: buffer_load_dword 324;CHECK: buffer_load_dword 325define amdgpu_ps void @test_set_inactive2(i32 inreg %idx0, i32 inreg %idx1) { 326main_body: 327 %src1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i1 0, i1 0) 328 %src1.0 = bitcast float %src1 to i32 329 %src1.1 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %src1.0, i32 undef) 330 %src0 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0) 331 %src0.0 = bitcast float %src0 to i32 332 %src0.1 = call i32 @llvm.amdgcn.wqm.i32(i32 %src0.0) 333 %out = add i32 %src0.1, %src1.1 334 %out.0 = bitcast i32 %out to float 335 call void @llvm.amdgcn.buffer.store.f32(float %out.0, <4 x i32> undef, i32 %idx1, i32 0, i1 0, i1 0) 336 ret void 337} 338 339; Check a case of one branch of an if-else requiring WQM, the other requiring 340; exact. 341; 342; Note: In this particular case, the save-and-restore could be avoided if the 343; analysis understood that the two branches of the if-else are mutually 344; exclusive. 345; 346;CHECK-LABEL: {{^}}test_control_flow_0: 347;CHECK-NEXT: ; %main_body 348;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 349;CHECK-NEXT: s_wqm_b64 exec, exec 350;CHECK: %ELSE 351;CHECK: s_and_saveexec_b64 [[SAVED:s\[[0-9]+:[0-9]+\]]], [[ORIG]] 352;CHECK: store 353;CHECK: s_mov_b64 exec, [[SAVED]] 354;CHECK: %IF 355;CHECK: image_sample 356;CHECK: image_sample 357define amdgpu_ps float @test_control_flow_0(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data) { 358main_body: 359 %cmp = icmp eq i32 %z, 0 360 br i1 %cmp, label %IF, label %ELSE 361 362IF: 363 %c.bc = bitcast i32 %c to float 364 %tex = call <4 x float> @llvm.amdgcn.image.sample.v4f32.f32.v8i32(float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0 365 %dtex = call <4 x float> @llvm.amdgcn.image.sample.v4f32.v4f32.v8i32(<4 x float> %tex, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0 366 %data.if = extractelement <4 x float> %dtex, i32 0 367 br label %END 368 369ELSE: 370 call void @llvm.amdgcn.buffer.store.f32(float %data, <4 x i32> undef, i32 %c, i32 0, i1 0, i1 0) 371 br label %END 372 373END: 374 %r = phi float [ %data.if, %IF ], [ %data, %ELSE ] 375 ret float %r 376} 377 378; Reverse branch order compared to the previous test. 379; 380;CHECK-LABEL: {{^}}test_control_flow_1: 381;CHECK-NEXT: ; %main_body 382;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 383;CHECK-NEXT: s_wqm_b64 exec, exec 384;CHECK: %IF 385;CHECK: image_sample 386;CHECK: image_sample 387;CHECK: %Flow 388;CHECK-NEXT: s_or_saveexec_b64 [[SAVED:s\[[0-9]+:[0-9]+\]]], 389;CHECK-NEXT: s_and_b64 exec, exec, [[ORIG]] 390;CHECK-NEXT: s_and_b64 [[SAVED]], exec, [[SAVED]] 391;CHECK-NEXT: s_xor_b64 exec, exec, [[SAVED]] 392;CHECK-NEXT: mask branch [[END_BB:BB[0-9]+_[0-9]+]] 393;CHECK-NEXT: BB{{[0-9]+_[0-9]+}}: ; %ELSE 394;CHECK: store_dword 395;CHECK: [[END_BB]]: ; %END 396;CHECK: s_or_b64 exec, exec, 397;CHECK: v_mov_b32_e32 v0 398;CHECK: ; return 399define amdgpu_ps float @test_control_flow_1(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data) { 400main_body: 401 %cmp = icmp eq i32 %z, 0 402 br i1 %cmp, label %ELSE, label %IF 403 404IF: 405 %c.bc = bitcast i32 %c to float 406 %tex = call <4 x float> @llvm.amdgcn.image.sample.v4f32.f32.v8i32(float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0 407 %dtex = call <4 x float> @llvm.amdgcn.image.sample.v4f32.v4f32.v8i32(<4 x float> %tex, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0 408 %data.if = extractelement <4 x float> %dtex, i32 0 409 br label %END 410 411ELSE: 412 call void @llvm.amdgcn.buffer.store.f32(float %data, <4 x i32> undef, i32 %c, i32 0, i1 0, i1 0) 413 br label %END 414 415END: 416 %r = phi float [ %data.if, %IF ], [ %data, %ELSE ] 417 ret float %r 418} 419 420; Check that branch conditions are properly marked as needing WQM... 421; 422;CHECK-LABEL: {{^}}test_control_flow_2: 423;CHECK-NEXT: ; %main_body 424;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 425;CHECK-NEXT: s_wqm_b64 exec, exec 426;CHECK: s_and_b64 exec, exec, [[ORIG]] 427;CHECK: store 428;CHECK: s_wqm_b64 exec, exec 429;CHECK: load 430;CHECK: s_and_b64 exec, exec, [[ORIG]] 431;CHECK: store 432;CHECK: s_wqm_b64 exec, exec 433;CHECK: v_cmp 434define amdgpu_ps <4 x float> @test_control_flow_2(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, <3 x i32> %idx, <2 x float> %data, i32 %coord) { 435main_body: 436 %idx.1 = extractelement <3 x i32> %idx, i32 0 437 %data.1 = extractelement <2 x float> %data, i32 0 438 call void @llvm.amdgcn.buffer.store.f32(float %data.1, <4 x i32> undef, i32 %idx.1, i32 0, i1 0, i1 0) 439 440 ; The load that determines the branch (and should therefore be WQM) is 441 ; surrounded by stores that require disabled WQM. 442 %idx.2 = extractelement <3 x i32> %idx, i32 1 443 %z = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx.2, i32 0, i1 0, i1 0) 444 445 %idx.3 = extractelement <3 x i32> %idx, i32 2 446 %data.3 = extractelement <2 x float> %data, i32 1 447 call void @llvm.amdgcn.buffer.store.f32(float %data.3, <4 x i32> undef, i32 %idx.3, i32 0, i1 0, i1 0) 448 449 %cc = fcmp ogt float %z, 0.0 450 br i1 %cc, label %IF, label %ELSE 451 452IF: 453 %coord.IF = mul i32 %coord, 3 454 br label %END 455 456ELSE: 457 %coord.ELSE = mul i32 %coord, 4 458 br label %END 459 460END: 461 %coord.END = phi i32 [ %coord.IF, %IF ], [ %coord.ELSE, %ELSE ] 462 %coord.END.bc = bitcast i32 %coord.END to float 463 %tex = call <4 x float> @llvm.amdgcn.image.sample.v4f32.f32.v8i32(float %coord.END.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0 464 ret <4 x float> %tex 465} 466 467; ... but only if they really do need it. 468; 469;CHECK-LABEL: {{^}}test_control_flow_3: 470;CHECK-NEXT: ; %main_body 471;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 472;CHECK-NEXT: s_wqm_b64 exec, exec 473;CHECK: image_sample 474;CHECK: s_and_b64 exec, exec, [[ORIG]] 475;CHECK: image_sample 476;CHECK-DAG: v_cmp 477;CHECK-DAG: store 478define amdgpu_ps float @test_control_flow_3(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %idx, float %coord) { 479main_body: 480 %tex = call <4 x float> @llvm.amdgcn.image.sample.v4f32.f32.v8i32(float %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0 481 %dtex = call <4 x float> @llvm.amdgcn.image.sample.v4f32.v4f32.v8i32(<4 x float> %tex, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0 482 %dtex.1 = extractelement <4 x float> %dtex, i32 0 483 call void @llvm.amdgcn.buffer.store.f32(float %dtex.1, <4 x i32> undef, i32 %idx, i32 0, i1 0, i1 0) 484 485 %cc = fcmp ogt float %dtex.1, 0.0 486 br i1 %cc, label %IF, label %ELSE 487 488IF: 489 %tex.IF = fmul float %dtex.1, 3.0 490 br label %END 491 492ELSE: 493 %tex.ELSE = fmul float %dtex.1, 4.0 494 br label %END 495 496END: 497 %tex.END = phi float [ %tex.IF, %IF ], [ %tex.ELSE, %ELSE ] 498 ret float %tex.END 499} 500 501; Another test that failed at some point because of terminator handling. 502; 503;CHECK-LABEL: {{^}}test_control_flow_4: 504;CHECK-NEXT: ; %main_body 505;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 506;CHECK-NEXT: s_wqm_b64 exec, exec 507;CHECK: %IF 508;CHECK: s_and_saveexec_b64 [[SAVE:s\[[0-9]+:[0-9]+\]]], [[ORIG]] 509;CHECK: load 510;CHECK: store 511;CHECK: s_mov_b64 exec, [[SAVE]] 512;CHECK: %END 513;CHECK: image_sample 514;CHECK: image_sample 515define amdgpu_ps <4 x float> @test_control_flow_4(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float %coord, i32 %y, float %z) { 516main_body: 517 %cond = icmp eq i32 %y, 0 518 br i1 %cond, label %IF, label %END 519 520IF: 521 %data = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 0, i32 0, i1 0, i1 0) 522 call void @llvm.amdgcn.buffer.store.f32(float %data, <4 x i32> undef, i32 1, i32 0, i1 0, i1 0) 523 br label %END 524 525END: 526 %tex = call <4 x float> @llvm.amdgcn.image.sample.v4f32.f32.v8i32(float %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0 527 %dtex = call <4 x float> @llvm.amdgcn.image.sample.v4f32.v4f32.v8i32(<4 x float> %tex, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0 528 ret <4 x float> %dtex 529} 530 531; Kill is performed in WQM mode so that uniform kill behaves correctly ... 532; 533;CHECK-LABEL: {{^}}test_kill_0: 534;CHECK-NEXT: ; %main_body 535;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 536;CHECK-NEXT: s_wqm_b64 exec, exec 537;CHECK: s_and_b64 exec, exec, [[ORIG]] 538;CHECK: image_sample 539;CHECK: buffer_store_dword 540;CHECK: s_wqm_b64 exec, exec 541;CHECK: v_cmpx_ 542;CHECK: s_and_saveexec_b64 [[SAVE:s\[[0-9]+:[0-9]+\]]], [[ORIG]] 543;CHECK: buffer_store_dword 544;CHECK: s_mov_b64 exec, [[SAVE]] 545;CHECK: image_sample 546define amdgpu_ps <4 x float> @test_kill_0(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float addrspace(1)* inreg %ptr, <2 x i32> %idx, <2 x float> %data, float %coord, float %coord2, float %z) { 547main_body: 548 %tex = call <4 x float> @llvm.amdgcn.image.sample.v4f32.f32.v8i32(float %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0 549 %idx.0 = extractelement <2 x i32> %idx, i32 0 550 %data.0 = extractelement <2 x float> %data, i32 0 551 call void @llvm.amdgcn.buffer.store.f32(float %data.0, <4 x i32> undef, i32 %idx.0, i32 0, i1 0, i1 0) 552 553 call void @llvm.AMDGPU.kill(float %z) 554 555 %idx.1 = extractelement <2 x i32> %idx, i32 1 556 %data.1 = extractelement <2 x float> %data, i32 1 557 call void @llvm.amdgcn.buffer.store.f32(float %data.1, <4 x i32> undef, i32 %idx.1, i32 0, i1 0, i1 0) 558 %tex2 = call <4 x float> @llvm.amdgcn.image.sample.v4f32.f32.v8i32(float %coord2, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0 559 %dtex = call <4 x float> @llvm.amdgcn.image.sample.v4f32.v4f32.v8i32(<4 x float> %tex2, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0 560 %out = fadd <4 x float> %tex, %dtex 561 562 ret <4 x float> %out 563} 564 565; ... but only if WQM is necessary. 566; 567; CHECK-LABEL: {{^}}test_kill_1: 568; CHECK-NEXT: ; %main_body 569; CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 570; CHECK: s_wqm_b64 exec, exec 571; CHECK: image_sample 572; CHECK: s_and_b64 exec, exec, [[ORIG]] 573; CHECK: image_sample 574; CHECK: buffer_store_dword 575; CHECK-NOT: wqm 576; CHECK: v_cmpx_ 577define amdgpu_ps <4 x float> @test_kill_1(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %idx, float %data, float %coord, float %coord2, float %z) { 578main_body: 579 %tex = call <4 x float> @llvm.amdgcn.image.sample.v4f32.f32.v8i32(float %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0 580 %dtex = call <4 x float> @llvm.amdgcn.image.sample.v4f32.v4f32.v8i32(<4 x float> %tex, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0 581 582 call void @llvm.amdgcn.buffer.store.f32(float %data, <4 x i32> undef, i32 0, i32 0, i1 0, i1 0) 583 584 call void @llvm.AMDGPU.kill(float %z) 585 586 ret <4 x float> %dtex 587} 588 589; Check prolog shaders. 590; 591; CHECK-LABEL: {{^}}test_prolog_1: 592; CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 593; CHECK: s_wqm_b64 exec, exec 594; CHECK: v_add_f32_e32 v0, 595; CHECK: s_and_b64 exec, exec, [[ORIG]] 596define amdgpu_ps float @test_prolog_1(float %a, float %b) #5 { 597main_body: 598 %s = fadd float %a, %b 599 ret float %s 600} 601 602; CHECK-LABEL: {{^}}test_loop_vcc: 603; CHECK-NEXT: ; %entry 604; CHECK-NEXT: s_mov_b64 [[LIVE:s\[[0-9]+:[0-9]+\]]], exec 605; CHECK: s_wqm_b64 exec, exec 606; CHECK: s_and_b64 exec, exec, [[LIVE]] 607; CHECK: image_store 608; CHECK: s_wqm_b64 exec, exec 609; CHECK-DAG: v_mov_b32_e32 [[CTR:v[0-9]+]], 0 610; CHECK-DAG: v_mov_b32_e32 [[SEVEN:v[0-9]+]], 0x40e00000 611 612; CHECK: [[LOOPHDR:BB[0-9]+_[0-9]+]]: ; %body 613; CHECK: v_add_f32_e32 [[CTR]], 2.0, [[CTR]] 614; CHECK: v_cmp_gt_f32_e32 vcc, [[CTR]], [[SEVEN]] 615; CHECK: s_cbranch_vccz [[LOOPHDR]] 616; CHECK: ; %break 617 618; CHECK: ; return 619define amdgpu_ps <4 x float> @test_loop_vcc(<4 x float> %in) nounwind { 620entry: 621 call void @llvm.amdgcn.image.store.v4f32.v4i32.v8i32(<4 x float> %in, <4 x i32> undef, <8 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0) 622 br label %loop 623 624loop: 625 %ctr.iv = phi float [ 0.0, %entry ], [ %ctr.next, %body ] 626 %c.iv = phi <4 x float> [ %in, %entry ], [ %c.next, %body ] 627 %cc = fcmp ogt float %ctr.iv, 7.0 628 br i1 %cc, label %break, label %body 629 630body: 631 %c.next = call <4 x float> @llvm.amdgcn.image.sample.v4f32.v4f32.v8i32(<4 x float> %c.iv, <8 x i32> undef, <4 x i32> undef, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0 632 %ctr.next = fadd float %ctr.iv, 2.0 633 br label %loop 634 635break: 636 ret <4 x float> %c.iv 637} 638 639; Only intrinsic stores need exact execution -- other stores do not have 640; externally visible effects and may require WQM for correctness. 641; 642; CHECK-LABEL: {{^}}test_alloca: 643; CHECK: s_mov_b64 [[LIVE:s\[[0-9]+:[0-9]+\]]], exec 644; CHECK: s_wqm_b64 exec, exec 645 646; CHECK: s_and_b64 exec, exec, [[LIVE]] 647; CHECK: buffer_store_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 648; CHECK: s_wqm_b64 exec, exec 649; CHECK: buffer_store_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offset:4{{$}} 650; CHECK: s_and_b64 exec, exec, [[LIVE]] 651; CHECK: buffer_store_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 idxen 652; CHECK: s_wqm_b64 exec, exec 653; CHECK: buffer_load_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen 654 655; CHECK: s_and_b64 exec, exec, [[LIVE]] 656; CHECK: image_sample 657; CHECK: buffer_store_dwordx4 658define amdgpu_ps void @test_alloca(float %data, i32 %a, i32 %idx) nounwind { 659entry: 660 %array = alloca [32 x i32], align 4 661 662 call void @llvm.amdgcn.buffer.store.f32(float %data, <4 x i32> undef, i32 0, i32 0, i1 0, i1 0) 663 664 %s.gep = getelementptr [32 x i32], [32 x i32]* %array, i32 0, i32 0 665 store volatile i32 %a, i32* %s.gep, align 4 666 667 call void @llvm.amdgcn.buffer.store.f32(float %data, <4 x i32> undef, i32 1, i32 0, i1 0, i1 0) 668 669 %c.gep = getelementptr [32 x i32], [32 x i32]* %array, i32 0, i32 %idx 670 %c = load i32, i32* %c.gep, align 4 671 %c.bc = bitcast i32 %c to float 672 %t = call <4 x float> @llvm.amdgcn.image.sample.v4f32.f32.v8i32(float %c.bc, <8 x i32> undef, <4 x i32> undef, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0 673 call void @llvm.amdgcn.buffer.store.v4f32(<4 x float> %t, <4 x i32> undef, i32 0, i32 0, i1 0, i1 0) 674 675 ret void 676} 677 678; Must return to exact at the end of a non-void returning shader, 679; otherwise the EXEC mask exported by the epilog will be wrong. This is true 680; even if the shader has no kills, because a kill could have happened in a 681; previous shader fragment. 682; 683; CHECK-LABEL: {{^}}test_nonvoid_return: 684; CHECK: s_mov_b64 [[LIVE:s\[[0-9]+:[0-9]+\]]], exec 685; CHECK: s_wqm_b64 exec, exec 686; 687; CHECK: s_and_b64 exec, exec, [[LIVE]] 688; CHECK-NOT: exec 689define amdgpu_ps <4 x float> @test_nonvoid_return() nounwind { 690 %tex = call <4 x float> @llvm.amdgcn.image.sample.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0 691 %dtex = call <4 x float> @llvm.amdgcn.image.sample.v4f32.v4f32.v8i32(<4 x float> %tex, <8 x i32> undef, <4 x i32> undef, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0 692 ret <4 x float> %dtex 693} 694 695; CHECK-LABEL: {{^}}test_nonvoid_return_unreachable: 696; CHECK: s_mov_b64 [[LIVE:s\[[0-9]+:[0-9]+\]]], exec 697; CHECK: s_wqm_b64 exec, exec 698; 699; CHECK: s_and_b64 exec, exec, [[LIVE]] 700; CHECK-NOT: exec 701define amdgpu_ps <4 x float> @test_nonvoid_return_unreachable(i32 inreg %c) nounwind { 702entry: 703 %tex = call <4 x float> @llvm.amdgcn.image.sample.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0 704 %dtex = call <4 x float> @llvm.amdgcn.image.sample.v4f32.v4f32.v8i32(<4 x float> %tex, <8 x i32> undef, <4 x i32> undef, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0 705 %cc = icmp sgt i32 %c, 0 706 br i1 %cc, label %if, label %else 707 708if: 709 store volatile <4 x float> %dtex, <4 x float> addrspace(1)* undef 710 unreachable 711 712else: 713 ret <4 x float> %dtex 714} 715 716; Test awareness that s_wqm_b64 clobbers SCC. 717; 718; CHECK-LABEL: {{^}}test_scc: 719; CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 720; CHECK: s_wqm_b64 exec, exec 721; CHECK: s_cmp_ 722; CHECK-NEXT: s_cbranch_scc 723; CHECK: ; %if 724; CHECK: s_and_b64 exec, exec, [[ORIG]] 725; CHECK: image_sample 726; CHECK: ; %else 727; CHECK: s_and_b64 exec, exec, [[ORIG]] 728; CHECK: image_sample 729; CHECK: ; %end 730define amdgpu_ps <4 x float> @test_scc(i32 inreg %sel, i32 %idx) #1 { 731main_body: 732 %cc = icmp sgt i32 %sel, 0 733 br i1 %cc, label %if, label %else 734 735if: 736 %r.if = call <4 x float> @llvm.amdgcn.image.sample.v4f32.f32.v8i32(float 0.0, <8 x i32> undef, <4 x i32> undef, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0 737 br label %end 738 739else: 740 %r.else = call <4 x float> @llvm.amdgcn.image.sample.v4f32.v2f32.v8i32(<2 x float> <float 0.0, float bitcast (i32 1 to float)>, <8 x i32> undef, <4 x i32> undef, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0 741 br label %end 742 743end: 744 %r = phi <4 x float> [ %r.if, %if ], [ %r.else, %else ] 745 call void @llvm.amdgcn.buffer.store.f32(float 1.0, <4 x i32> undef, i32 %idx, i32 0, i1 0, i1 0) 746 ret <4 x float> %r 747} 748 749declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #1 750declare void @llvm.amdgcn.image.store.v4f32.v4i32.v8i32(<4 x float>, <4 x i32>, <8 x i32>, i32, i1, i1, i1, i1) #1 751declare void @llvm.amdgcn.buffer.store.f32(float, <4 x i32>, i32, i32, i1, i1) #2 752declare void @llvm.amdgcn.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i1, i1) #2 753declare <4 x float> @llvm.amdgcn.image.load.v4f32.v4i32.v8i32(<4 x i32>, <8 x i32>, i32, i1, i1, i1, i1) #3 754declare float @llvm.amdgcn.buffer.load.f32(<4 x i32>, i32, i32, i1, i1) #3 755declare <4 x float> @llvm.amdgcn.image.sample.v4f32.f32.v8i32(float, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #3 756declare <4 x float> @llvm.amdgcn.image.sample.v4f32.v2f32.v8i32(<2 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #3 757declare <4 x float> @llvm.amdgcn.image.sample.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #3 758declare void @llvm.AMDGPU.kill(float) #1 759declare float @llvm.amdgcn.wqm.f32(float) #3 760declare i32 @llvm.amdgcn.wqm.i32(i32) #3 761declare float @llvm.amdgcn.wwm.f32(float) #3 762declare i32 @llvm.amdgcn.wwm.i32(i32) #3 763declare i32 @llvm.amdgcn.set.inactive.i32(i32, i32) #4 764declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #3 765declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32) #3 766declare <2 x half> @llvm.amdgcn.cvt.pkrtz(float, float) #3 767declare void @llvm.amdgcn.exp.compr.v2f16(i32, i32, <2 x half>, <2 x half>, i1, i1) #1 768declare float @llvm.amdgcn.interp.p1(float, i32, i32, i32) #2 769declare float @llvm.amdgcn.interp.p2(float, float, i32, i32, i32) #2 770 771attributes #1 = { nounwind } 772attributes #2 = { nounwind readonly } 773attributes #3 = { nounwind readnone } 774attributes #4 = { nounwind readnone convergent } 775attributes #5 = { "amdgpu-ps-wqm-outputs" } 776attributes #6 = { nounwind "InitialPSInputAddr"="2" } 777