1;RUN: llc < %s -march=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck %s --check-prefix=CHECK --check-prefix=SI 2;RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s --check-prefix=CHECK --check-prefix=VI 3 4; Check that WQM isn't triggered by image load/store intrinsics. 5; 6;CHECK-LABEL: {{^}}test1: 7;CHECK-NOT: s_wqm 8define <4 x float> @test1(<8 x i32> inreg %rsrc, <4 x i32> %c) #0 { 9main_body: 10 %tex = call <4 x float> @llvm.amdgcn.image.load.v4i32(<4 x i32> %c, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) 11 call void @llvm.amdgcn.image.store.v4i32(<4 x float> %tex, <4 x i32> %c, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) 12 ret <4 x float> %tex 13} 14 15; Check that WQM is triggered by image samples and left untouched for loads... 16; 17;CHECK-LABEL: {{^}}test2: 18;CHECK-NEXT: ; %main_body 19;CHECK-NEXT: s_wqm_b64 exec, exec 20;CHECK: image_sample 21;CHECK-NOT: exec 22;CHECK: _load_dword v0, 23define float @test2(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float addrspace(1)* inreg %ptr, <4 x i32> %c) #0 { 24main_body: 25 %c.1 = call <4 x float> @llvm.SI.image.sample.v4i32(<4 x i32> %c, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) 26 %c.2 = bitcast <4 x float> %c.1 to <4 x i32> 27 %c.3 = extractelement <4 x i32> %c.2, i32 0 28 %gep = getelementptr float, float addrspace(1)* %ptr, i32 %c.3 29 %data = load float, float addrspace(1)* %gep 30 ret float %data 31} 32 33; ... but disabled for stores (and, in this simple case, not re-enabled). 34; 35;CHECK-LABEL: {{^}}test3: 36;CHECK-NEXT: ; %main_body 37;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 38;CHECK-NEXT: s_wqm_b64 exec, exec 39;CHECK: image_sample 40;CHECK: s_and_b64 exec, exec, [[ORIG]] 41;CHECK: store 42;CHECK-NOT: exec 43define <4 x float> @test3(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float addrspace(1)* inreg %ptr, <4 x i32> %c) #0 { 44main_body: 45 %tex = call <4 x float> @llvm.SI.image.sample.v4i32(<4 x i32> %c, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) 46 %tex.1 = bitcast <4 x float> %tex to <4 x i32> 47 %tex.2 = extractelement <4 x i32> %tex.1, i32 0 48 %gep = getelementptr float, float addrspace(1)* %ptr, i32 %tex.2 49 %wr = extractelement <4 x float> %tex, i32 1 50 store float %wr, float addrspace(1)* %gep 51 ret <4 x float> %tex 52} 53 54; Check that WQM is re-enabled when required. 55; 56;CHECK-LABEL: {{^}}test4: 57;CHECK-NEXT: ; %main_body 58;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 59;CHECK-NEXT: s_wqm_b64 exec, exec 60;CHECK: v_mul_lo_i32 [[MUL:v[0-9]+]], v0, v1 61;CHECK: s_and_b64 exec, exec, [[ORIG]] 62;CHECK: store 63;CHECK: s_wqm_b64 exec, exec 64;CHECK: image_sample v[0:3], [[MUL]], s[0:7], s[8:11] dmask:0xf 65define <4 x float> @test4(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float addrspace(1)* inreg %ptr, i32 %c, i32 %d, float %data) #0 { 66main_body: 67 %c.1 = mul i32 %c, %d 68 %gep = getelementptr float, float addrspace(1)* %ptr, i32 %c.1 69 store float %data, float addrspace(1)* %gep 70 %tex = call <4 x float> @llvm.SI.image.sample.i32(i32 %c.1, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) 71 ret <4 x float> %tex 72} 73 74; Check a case of one branch of an if-else requiring WQM, the other requiring 75; exact. 76; 77; Note: In this particular case, the save-and-restore could be avoided if the 78; analysis understood that the two branches of the if-else are mutually 79; exclusive. 80; 81;CHECK-LABEL: {{^}}test_control_flow_0: 82;CHECK-NEXT: ; %main_body 83;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 84;CHECK-NEXT: s_wqm_b64 exec, exec 85;CHECK: %ELSE 86;CHECK: s_and_saveexec_b64 [[SAVED:s\[[0-9]+:[0-9]+\]]], [[ORIG]] 87;CHECK: store 88;CHECK: s_mov_b64 exec, [[SAVED]] 89;CHECK: %IF 90;CHECK: image_sample 91define float @test_control_flow_0(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float addrspace(1)* inreg %ptr, i32 %c, i32 %z, float %data) #0 { 92main_body: 93 %cmp = icmp eq i32 %z, 0 94 br i1 %cmp, label %IF, label %ELSE 95 96IF: 97 %tex = call <4 x float> @llvm.SI.image.sample.i32(i32 %c, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) 98 %data.if = extractelement <4 x float> %tex, i32 0 99 br label %END 100 101ELSE: 102 %gep = getelementptr float, float addrspace(1)* %ptr, i32 %c 103 store float %data, float addrspace(1)* %gep 104 br label %END 105 106END: 107 %r = phi float [ %data.if, %IF ], [ %data, %ELSE ] 108 ret float %r 109} 110 111; Reverse branch order compared to the previous test. 112; 113;CHECK-LABEL: {{^}}test_control_flow_1: 114;CHECK-NEXT: ; %main_body 115;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 116;CHECK-NEXT: s_wqm_b64 exec, exec 117;CHECK: %IF 118;CHECK: image_sample 119;CHECK: %Flow 120;CHECK-NEXT: s_or_saveexec_b64 [[SAVED:s\[[0-9]+:[0-9]+\]]], 121;CHECK-NEXT: s_and_b64 exec, exec, [[ORIG]] 122;CHECK-NEXT: s_and_b64 [[SAVED]], exec, [[SAVED]] 123;CHECK-NEXT: s_xor_b64 exec, exec, [[SAVED]] 124;CHECK-NEXT: %ELSE 125;CHECK: store 126;CHECK: %END 127define float @test_control_flow_1(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float addrspace(1)* inreg %ptr, i32 %c, i32 %z, float %data) #0 { 128main_body: 129 %cmp = icmp eq i32 %z, 0 130 br i1 %cmp, label %ELSE, label %IF 131 132IF: 133 %tex = call <4 x float> @llvm.SI.image.sample.i32(i32 %c, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) 134 %data.if = extractelement <4 x float> %tex, i32 0 135 br label %END 136 137ELSE: 138 %gep = getelementptr float, float addrspace(1)* %ptr, i32 %c 139 store float %data, float addrspace(1)* %gep 140 br label %END 141 142END: 143 %r = phi float [ %data.if, %IF ], [ %data, %ELSE ] 144 ret float %r 145} 146 147; Check that branch conditions are properly marked as needing WQM... 148; 149;CHECK-LABEL: {{^}}test_control_flow_2: 150;CHECK-NEXT: ; %main_body 151;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 152;CHECK-NEXT: s_wqm_b64 exec, exec 153;CHECK: s_and_b64 exec, exec, [[ORIG]] 154;CHECK: store 155;CHECK: s_wqm_b64 exec, exec 156;CHECK: load 157;CHECK: s_and_b64 exec, exec, [[ORIG]] 158;CHECK: store 159;CHECK: s_wqm_b64 exec, exec 160;CHECK: v_cmp 161define <4 x float> @test_control_flow_2(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float addrspace(1)* inreg %ptr, <3 x i32> %idx, <2 x float> %data, i32 %coord) #0 { 162main_body: 163 %idx.1 = extractelement <3 x i32> %idx, i32 0 164 %gep.1 = getelementptr float, float addrspace(1)* %ptr, i32 %idx.1 165 %data.1 = extractelement <2 x float> %data, i32 0 166 store float %data.1, float addrspace(1)* %gep.1 167 168 ; The load that determines the branch (and should therefore be WQM) is 169 ; surrounded by stores that require disabled WQM. 170 %idx.2 = extractelement <3 x i32> %idx, i32 1 171 %gep.2 = getelementptr float, float addrspace(1)* %ptr, i32 %idx.2 172 %z = load float, float addrspace(1)* %gep.2 173 174 %idx.3 = extractelement <3 x i32> %idx, i32 2 175 %gep.3 = getelementptr float, float addrspace(1)* %ptr, i32 %idx.3 176 %data.3 = extractelement <2 x float> %data, i32 1 177 store float %data.3, float addrspace(1)* %gep.3 178 179 %cc = fcmp ogt float %z, 0.0 180 br i1 %cc, label %IF, label %ELSE 181 182IF: 183 %coord.IF = mul i32 %coord, 3 184 br label %END 185 186ELSE: 187 %coord.ELSE = mul i32 %coord, 4 188 br label %END 189 190END: 191 %coord.END = phi i32 [ %coord.IF, %IF ], [ %coord.ELSE, %ELSE ] 192 %tex = call <4 x float> @llvm.SI.image.sample.i32(i32 %coord.END, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) 193 ret <4 x float> %tex 194} 195 196; ... but only if they really do need it. 197; 198;CHECK-LABEL: {{^}}test_control_flow_3: 199;CHECK-NEXT: ; %main_body 200;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 201;CHECK-NEXT: s_wqm_b64 exec, exec 202;CHECK: image_sample 203;CHECK: s_and_b64 exec, exec, [[ORIG]] 204;CHECK: store 205;CHECK: load 206;CHECK: store 207;CHECK: v_cmp 208define float @test_control_flow_3(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float addrspace(1)* inreg %ptr, <3 x i32> %idx, <2 x float> %data, i32 %coord) #0 { 209main_body: 210 %tex = call <4 x float> @llvm.SI.image.sample.i32(i32 %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) 211 %tex.1 = extractelement <4 x float> %tex, i32 0 212 213 %idx.1 = extractelement <3 x i32> %idx, i32 0 214 %gep.1 = getelementptr float, float addrspace(1)* %ptr, i32 %idx.1 215 %data.1 = extractelement <2 x float> %data, i32 0 216 store float %data.1, float addrspace(1)* %gep.1 217 218 %idx.2 = extractelement <3 x i32> %idx, i32 1 219 %gep.2 = getelementptr float, float addrspace(1)* %ptr, i32 %idx.2 220 %z = load float, float addrspace(1)* %gep.2 221 222 %idx.3 = extractelement <3 x i32> %idx, i32 2 223 %gep.3 = getelementptr float, float addrspace(1)* %ptr, i32 %idx.3 224 %data.3 = extractelement <2 x float> %data, i32 1 225 store float %data.3, float addrspace(1)* %gep.3 226 227 %cc = fcmp ogt float %z, 0.0 228 br i1 %cc, label %IF, label %ELSE 229 230IF: 231 %tex.IF = fmul float %tex.1, 3.0 232 br label %END 233 234ELSE: 235 %tex.ELSE = fmul float %tex.1, 4.0 236 br label %END 237 238END: 239 %tex.END = phi float [ %tex.IF, %IF ], [ %tex.ELSE, %ELSE ] 240 ret float %tex.END 241} 242 243; Another test that failed at some point because of terminator handling. 244; 245;CHECK-LABEL: {{^}}test_control_flow_4: 246;CHECK-NEXT: ; %main_body 247;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 248;CHECK-NEXT: s_wqm_b64 exec, exec 249;CHECK: %IF 250;CHECK: load 251;CHECK: s_and_saveexec_b64 [[SAVE:s\[[0-9]+:[0-9]+\]]], [[ORIG]] 252;CHECK: store 253;CHECK: s_mov_b64 exec, [[SAVE]] 254;CHECK: %END 255;CHECK: image_sample 256define <4 x float> @test_control_flow_4(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float addrspace(1)* inreg %ptr, i32 %coord, i32 %y, float %z) #0 { 257main_body: 258 %cond = icmp eq i32 %y, 0 259 br i1 %cond, label %IF, label %END 260 261IF: 262 %data = load float, float addrspace(1)* %ptr 263 %gep = getelementptr float, float addrspace(1)* %ptr, i32 1 264 store float %data, float addrspace(1)* %gep 265 br label %END 266 267END: 268 %tex = call <4 x float> @llvm.SI.image.sample.i32(i32 %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) 269 ret <4 x float> %tex 270} 271 272; Kill is performed in WQM mode so that uniform kill behaves correctly ... 273; 274;CHECK-LABEL: {{^}}test_kill_0: 275;CHECK-NEXT: ; %main_body 276;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 277;CHECK-NEXT: s_wqm_b64 exec, exec 278;CHECK: image_sample 279;CHECK: s_and_b64 exec, exec, [[ORIG]] 280;SI: buffer_store_dword 281;VI: flat_store_dword 282;CHECK: s_wqm_b64 exec, exec 283;CHECK: v_cmpx_ 284;CHECK: s_and_saveexec_b64 [[SAVE:s\[[0-9]+:[0-9]+\]]], [[ORIG]] 285;SI: buffer_store_dword 286;VI: flat_store_dword 287;CHECK: s_mov_b64 exec, [[SAVE]] 288;CHECK: image_sample 289define <4 x float> @test_kill_0(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float addrspace(1)* inreg %ptr, <2 x i32> %idx, <2 x float> %data, i32 %coord, i32 %coord2, float %z) #0 { 290main_body: 291 %tex = call <4 x float> @llvm.SI.image.sample.i32(i32 %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) 292 293 %idx.0 = extractelement <2 x i32> %idx, i32 0 294 %gep.0 = getelementptr float, float addrspace(1)* %ptr, i32 %idx.0 295 %data.0 = extractelement <2 x float> %data, i32 0 296 store float %data.0, float addrspace(1)* %gep.0 297 298 call void @llvm.AMDGPU.kill(float %z) 299 300 %idx.1 = extractelement <2 x i32> %idx, i32 1 301 %gep.1 = getelementptr float, float addrspace(1)* %ptr, i32 %idx.1 302 %data.1 = extractelement <2 x float> %data, i32 1 303 store float %data.1, float addrspace(1)* %gep.1 304 305 %tex2 = call <4 x float> @llvm.SI.image.sample.i32(i32 %coord2, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) 306 %out = fadd <4 x float> %tex, %tex2 307 308 ret <4 x float> %out 309} 310 311; ... but only if WQM is necessary. 312; 313;CHECK-LABEL: {{^}}test_kill_1: 314;CHECK-NEXT: ; %main_body 315;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec 316;CHECK-NEXT: s_wqm_b64 exec, exec 317;CHECK: image_sample 318;CHECK: s_and_b64 exec, exec, [[ORIG]] 319;SI: buffer_store_dword 320;VI: flat_store_dword 321;CHECK-NOT: wqm 322;CHECK: v_cmpx_ 323define <4 x float> @test_kill_1(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float addrspace(1)* inreg %ptr, i32 %idx, float %data, i32 %coord, i32 %coord2, float %z) #0 { 324main_body: 325 %tex = call <4 x float> @llvm.SI.image.sample.i32(i32 %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) 326 327 %gep = getelementptr float, float addrspace(1)* %ptr, i32 %idx 328 store float %data, float addrspace(1)* %gep 329 330 call void @llvm.AMDGPU.kill(float %z) 331 332 ret <4 x float> %tex 333} 334 335declare void @llvm.amdgcn.image.store.v4i32(<4 x float>, <4 x i32>, <8 x i32>, i32, i1, i1, i1, i1) #1 336 337declare <4 x float> @llvm.amdgcn.image.load.v4i32(<4 x i32>, <8 x i32>, i32, i1, i1, i1, i1) #2 338 339declare <4 x float> @llvm.SI.image.sample.i32(i32, <8 x i32>, <4 x i32>, i32, i32, i32, i32, i32, i32, i32, i32) #3 340declare <4 x float> @llvm.SI.image.sample.v4i32(<4 x i32>, <8 x i32>, <4 x i32>, i32, i32, i32, i32, i32, i32, i32, i32) #3 341 342declare void @llvm.AMDGPU.kill(float) 343declare void @llvm.SI.export(i32, i32, i32, i32, i32, float, float, float, float) 344 345attributes #0 = { "ShaderType"="0" } 346attributes #1 = { nounwind } 347attributes #2 = { nounwind readonly } 348attributes #3 = { nounwind readnone } 349