1; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=SICIVI -check-prefix=SICI -check-prefix=SIVIGFX9_10 %s 2; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=CI -check-prefix=GCN -check-prefix=SICIVI -check-prefix=SICI %s 3; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=VI -check-prefix=GCN -check-prefix=SICIVI -check-prefix=VIGFX9_10 -check-prefix=SIVIGFX9_10 %s 4; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=GFX9 -check-prefix=GFX9_10 -check-prefix=GCN -check-prefix=VIGFX9_10 -check-prefix=SIVIGFX9_10 %s 5; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=GFX10 -check-prefix=GFX9_10 -check-prefix=GCN -check-prefix=VIGFX9_10 -check-prefix=SIVIGFX9_10 %s 6 7; SMRD load with an immediate offset. 8; GCN-LABEL: {{^}}smrd0: 9; SICI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x1 ; encoding: [0x01 10; VIGFX9_10: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x4 11define amdgpu_kernel void @smrd0(i32 addrspace(1)* %out, i32 addrspace(4)* %ptr) #0 { 12entry: 13 %tmp = getelementptr i32, i32 addrspace(4)* %ptr, i64 1 14 %tmp1 = load i32, i32 addrspace(4)* %tmp 15 store i32 %tmp1, i32 addrspace(1)* %out 16 ret void 17} 18 19; SMRD load with the largest possible immediate offset. 20; GCN-LABEL: {{^}}smrd1: 21; SICI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0xff ; encoding: [0xff,0x{{[0-9]+[137]}} 22; VIGFX9_10: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x3fc 23define amdgpu_kernel void @smrd1(i32 addrspace(1)* %out, i32 addrspace(4)* %ptr) #0 { 24entry: 25 %tmp = getelementptr i32, i32 addrspace(4)* %ptr, i64 255 26 %tmp1 = load i32, i32 addrspace(4)* %tmp 27 store i32 %tmp1, i32 addrspace(1)* %out 28 ret void 29} 30 31; SMRD load with an offset greater than the largest possible immediate. 32; GCN-LABEL: {{^}}smrd2: 33; SI: s_movk_i32 s[[OFFSET:[0-9]]], 0x400 34; SI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], s[[OFFSET]] ; encoding: [0x0[[OFFSET]] 35; CI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x100 36; VIGFX9_10: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x400 37; GCN: s_endpgm 38define amdgpu_kernel void @smrd2(i32 addrspace(1)* %out, i32 addrspace(4)* %ptr) #0 { 39entry: 40 %tmp = getelementptr i32, i32 addrspace(4)* %ptr, i64 256 41 %tmp1 = load i32, i32 addrspace(4)* %tmp 42 store i32 %tmp1, i32 addrspace(1)* %out 43 ret void 44} 45 46; SMRD load with a 64-bit offset 47; GCN-LABEL: {{^}}smrd3: 48; FIXME: There are too many copies here because we don't fold immediates 49; through REG_SEQUENCE 50; SI: s_load_dwordx2 s[{{[0-9]:[0-9]}}], s[{{[0-9]:[0-9]}}], 0x13 ; encoding: [0x13 51; TODO: Add VI checks 52; GCN: s_endpgm 53define amdgpu_kernel void @smrd3(i32 addrspace(1)* %out, [8 x i32], i32 addrspace(4)* %ptr) #0 { 54entry: 55 %tmp = getelementptr i32, i32 addrspace(4)* %ptr, i64 4294967296 56 %tmp1 = load i32, i32 addrspace(4)* %tmp 57 store i32 %tmp1, i32 addrspace(1)* %out 58 ret void 59} 60 61; SMRD load with the largest possible immediate offset on VI 62; GCN-LABEL: {{^}}smrd4: 63; SI: s_mov_b32 [[OFFSET:s[0-9]+]], 0xffffc 64; SI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], [[OFFSET]] 65; CI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x3ffff 66; VI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0xffffc 67; GFX9_10: s_mov_b32 [[OFFSET:s[0-9]+]], 0xffffc 68; GFX9_10: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], [[OFFSET]] 69define amdgpu_kernel void @smrd4(i32 addrspace(1)* %out, i32 addrspace(4)* %ptr) #0 { 70entry: 71 %tmp = getelementptr i32, i32 addrspace(4)* %ptr, i64 262143 72 %tmp1 = load i32, i32 addrspace(4)* %tmp 73 store i32 %tmp1, i32 addrspace(1)* %out 74 ret void 75} 76 77; SMRD load with an offset greater than the largest possible immediate on VI 78; GCN-LABEL: {{^}}smrd5: 79; SIVIGFX9_10: s_mov_b32 [[OFFSET:s[0-9]+]], 0x100000 80; SIVIGFX9_10: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], [[OFFSET]] 81; CI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x40000 82; GCN: s_endpgm 83define amdgpu_kernel void @smrd5(i32 addrspace(1)* %out, i32 addrspace(4)* %ptr) #0 { 84entry: 85 %tmp = getelementptr i32, i32 addrspace(4)* %ptr, i64 262144 86 %tmp1 = load i32, i32 addrspace(4)* %tmp 87 store i32 %tmp1, i32 addrspace(1)* %out 88 ret void 89} 90 91; GFX9_10 can use a signed immediate byte offset 92; GCN-LABEL: {{^}}smrd6: 93; SICIVI: s_add_u32 s{{[0-9]}}, s{{[0-9]}}, -4 94; SICIVI: s_load_dword s{{[0-9]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0 95; GFX9_10: s_load_dword s{{[0-9]}}, s{{\[[0-9]+:[0-9]+\]}}, -0x4 96define amdgpu_kernel void @smrd6(i32 addrspace(1)* %out, i32 addrspace(4)* %ptr) #0 { 97entry: 98 %tmp = getelementptr i32, i32 addrspace(4)* %ptr, i64 -1 99 %tmp1 = load i32, i32 addrspace(4)* %tmp 100 store i32 %tmp1, i32 addrspace(1)* %out 101 ret void 102} 103 104; Don't use a negative SGPR offset 105; GCN-LABEL: {{^}}smrd7: 106; GCN: s_add_u32 s{{[0-9]}}, s{{[0-9]}}, 0xffe00000 107; SICIVI: s_load_dword s{{[0-9]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0 108; GFX9_10: s_load_dword s{{[0-9]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0 109define amdgpu_kernel void @smrd7(i32 addrspace(1)* %out, i32 addrspace(4)* %ptr) #0 { 110entry: 111 %tmp = getelementptr i32, i32 addrspace(4)* %ptr, i64 -524288 112 %tmp1 = load i32, i32 addrspace(4)* %tmp 113 store i32 %tmp1, i32 addrspace(1)* %out 114 ret void 115} 116 117; GCN-LABEL: {{^}}smrd_hazard: 118; GCN-DAG: s_mov_b32 s3, 3 119; GCN-DAG: s_mov_b32 s2, 2 120; GCN-DAG: s_mov_b32 s1, 1 121; GCN-DAG: s_mov_b32 s0, 0 122; SI-NEXT: nop 3 123; GFX10-NEXT: ; implicit-def: $vcc_hi 124; GCN-NEXT: s_buffer_load_dword s0, s[0:3], 0x0 125define amdgpu_ps float @smrd_hazard(<4 x i32> inreg %desc) #0 { 126main_body: 127 %d0 = insertelement <4 x i32> undef, i32 0, i32 0 128 %d1 = insertelement <4 x i32> %d0, i32 1, i32 1 129 %d2 = insertelement <4 x i32> %d1, i32 2, i32 2 130 %d3 = insertelement <4 x i32> %d2, i32 3, i32 3 131 %r = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %d3, i32 0, i32 0) 132 ret float %r 133} 134 135; SMRD load using the load.const.v4i32 intrinsic with an immediate offset 136; GCN-LABEL: {{^}}smrd_load_const0: 137; SICI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x4 ; encoding: [0x04 138; VIGFX9_10: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x10 139define amdgpu_ps void @smrd_load_const0(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in) #0 { 140main_body: 141 %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg, i32 0 142 %tmp20 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp 143 %tmp21 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 16, i32 0) 144 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %tmp21, float %tmp21, float %tmp21, float %tmp21, i1 true, i1 true) #0 145 ret void 146} 147 148; SMRD load using the load.const.v4i32 intrinsic with the largest possible immediate 149; offset. 150; GCN-LABEL: {{^}}smrd_load_const1: 151; SICI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0xff ; encoding: [0xff 152; SICI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0xff glc ; encoding: [0xff 153; VIGFX9_10-DAG: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x3fc ; 154; VIGFX9_10-DAG: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x3fc glc ; 155define amdgpu_ps void @smrd_load_const1(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in) #0 { 156main_body: 157 %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg, i32 0 158 %tmp20 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp 159 %tmp21 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 1020, i32 0) 160 %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in 161 %s.buffer = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %tmp22, i32 1020, i32 1) 162 %s.buffer.float = bitcast i32 %s.buffer to float 163 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %tmp21, float %tmp21, float %tmp21, float %s.buffer.float, i1 true, i1 true) #0 164 ret void 165} 166 167; SMRD load using the load.const.v4i32 intrinsic with an offset greater than the 168; largets possible immediate. 169; immediate offset. 170; GCN-LABEL: {{^}}smrd_load_const2: 171; SI: s_movk_i32 s[[OFFSET:[0-9]]], 0x400 172; SI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], s[[OFFSET]] ; encoding: [0x0[[OFFSET]] 173; SI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], s[[OFFSET]] ; encoding: [0x0[[OFFSET]] 174; CI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x100 175; CI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x100 176; VIGFX9_10: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x400 177; VIGFX9_10: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x400 178define amdgpu_ps void @smrd_load_const2(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in) #0 { 179main_body: 180 %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg, i32 0 181 %tmp20 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp 182 %tmp21 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 1024, i32 0) 183 %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in 184 %s.buffer = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %tmp22, i32 1024, i32 0) 185 %s.buffer.float = bitcast i32 %s.buffer to float 186 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %tmp21, float %tmp21, float %tmp21, float %s.buffer.float, i1 true, i1 true) #0 187 ret void 188} 189 190; SMRD load with the largest possible immediate offset on VI 191; GCN-LABEL: {{^}}smrd_load_const3: 192; SI: s_mov_b32 [[OFFSET:s[0-9]+]], 0xffffc 193; SI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], [[OFFSET]] 194; SI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], [[OFFSET]] 195; CI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x3ffff 196; CI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x3ffff 197; VIGFX9_10: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0xffffc 198; VIGFX9_10: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0xffffc 199define amdgpu_ps void @smrd_load_const3(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in) #0 { 200main_body: 201 %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg, i32 0 202 %tmp20 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp 203 %tmp21 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 1048572, i32 0) 204 %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in 205 %s.buffer = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %tmp22, i32 1048572, i32 0) 206 %s.buffer.float = bitcast i32 %s.buffer to float 207 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %tmp21, float %tmp21, float %tmp21, float %s.buffer.float, i1 true, i1 true) #0 208 ret void 209} 210 211; SMRD load with an offset greater than the largest possible immediate on VI 212; GCN-LABEL: {{^}}smrd_load_const4: 213; SIVIGFX9_10: s_mov_b32 [[OFFSET:s[0-9]+]], 0x100000 214; SIVIGFX9_10: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], [[OFFSET]] 215; SIVIGFX9_10: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], [[OFFSET]] 216; CI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x40000 217; CI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x40000 218; GCN: s_endpgm 219define amdgpu_ps void @smrd_load_const4(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in) #0 { 220main_body: 221 %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg, i32 0 222 %tmp20 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp 223 %tmp21 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 1048576, i32 0) 224 %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in 225 %s.buffer = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %tmp22, i32 1048576, i32 0) 226 %s.buffer.float = bitcast i32 %s.buffer to float 227 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %tmp21, float %tmp21, float %tmp21, float %s.buffer.float, i1 true, i1 true) #0 228 ret void 229} 230 231; dwordx2 s.buffer.load 232; GCN-LABEL: {{^}}s_buffer_load_dwordx2: 233; VIGFX9_10: s_buffer_load_dwordx2 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]:[0-9]}}], 0x80 234; SICI: s_buffer_load_dwordx2 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]:[0-9]}}], 0x20 235define amdgpu_ps void @s_buffer_load_dwordx2(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in) #0 { 236main_body: 237 %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in 238 %s.buffer = call <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32> %tmp22, i32 128, i32 0) 239 %s.buffer.0 = extractelement <2 x i32> %s.buffer, i32 0 240 %s.buffer.0.float = bitcast i32 %s.buffer.0 to float 241 %s.buffer.1 = extractelement <2 x i32> %s.buffer, i32 1 242 %s.buffer.1.float = bitcast i32 %s.buffer.1 to float 243 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %s.buffer.0.float, float %s.buffer.1.float, float %s.buffer.0.float, float %s.buffer.1.float, i1 true, i1 true) #0 244 ret void 245} 246 247; dwordx4 s.buffer.load 248; GCN-LABEL: {{^}}s_buffer_load_dwordx4: 249; VIGFX9_10: s_buffer_load_dwordx4 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]:[0-9]}}], 0x80 250; SICI: s_buffer_load_dwordx4 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]:[0-9]}}], 0x20 251define amdgpu_ps void @s_buffer_load_dwordx4(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in) #0 { 252main_body: 253 %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in 254 %s.buffer = call <4 x i32> @llvm.amdgcn.s.buffer.load.v4i32(<4 x i32> %tmp22, i32 128, i32 0) 255 %s.buffer.0 = extractelement <4 x i32> %s.buffer, i32 0 256 %s.buffer.0.float = bitcast i32 %s.buffer.0 to float 257 %s.buffer.1 = extractelement <4 x i32> %s.buffer, i32 1 258 %s.buffer.1.float = bitcast i32 %s.buffer.1 to float 259 %s.buffer.2 = extractelement <4 x i32> %s.buffer, i32 2 260 %s.buffer.2.float = bitcast i32 %s.buffer.2 to float 261 %s.buffer.3 = extractelement <4 x i32> %s.buffer, i32 3 262 %s.buffer.3.float = bitcast i32 %s.buffer.3 to float 263 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %s.buffer.0.float, float %s.buffer.1.float, float %s.buffer.2.float, float %s.buffer.3.float, i1 true, i1 true) #0 264 ret void 265} 266 267; dwordx8 s.buffer.load 268; GCN-LABEL: {{^}}s_buffer_load_dwordx8: 269; VIGFX9_10: s_buffer_load_dwordx8 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]:[0-9]}}], 0x80 270; SICI: s_buffer_load_dwordx8 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]:[0-9]}}], 0x20 271define amdgpu_ps void @s_buffer_load_dwordx8(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in) #0 { 272main_body: 273 %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in 274 %s.buffer = call <8 x i32> @llvm.amdgcn.s.buffer.load.v8i32(<4 x i32> %tmp22, i32 128, i32 0) 275 %s.buffer.0 = extractelement <8 x i32> %s.buffer, i32 0 276 %s.buffer.0.float = bitcast i32 %s.buffer.0 to float 277 %s.buffer.1 = extractelement <8 x i32> %s.buffer, i32 2 278 %s.buffer.1.float = bitcast i32 %s.buffer.1 to float 279 %s.buffer.2 = extractelement <8 x i32> %s.buffer, i32 5 280 %s.buffer.2.float = bitcast i32 %s.buffer.2 to float 281 %s.buffer.3 = extractelement <8 x i32> %s.buffer, i32 7 282 %s.buffer.3.float = bitcast i32 %s.buffer.3 to float 283 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %s.buffer.0.float, float %s.buffer.1.float, float %s.buffer.2.float, float %s.buffer.3.float, i1 true, i1 true) #0 284 ret void 285} 286 287; dwordx8 s.buffer.load 288; GCN-LABEL: {{^}}s_buffer_load_dwordx8_v8f32: 289; VIGFX9_10: s_buffer_load_dwordx8 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]:[0-9]}}], 0x80 290; SICI: s_buffer_load_dwordx8 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]:[0-9]}}], 0x20 291define amdgpu_ps void @s_buffer_load_dwordx8_v8f32(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in) #0 { 292main_body: 293 %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in 294 %s.buffer = call <8 x float> @llvm.amdgcn.s.buffer.load.v8f32(<4 x i32> %tmp22, i32 128, i32 0) 295 %s.buffer.0 = extractelement <8 x float> %s.buffer, i32 0 296 %s.buffer.1 = extractelement <8 x float> %s.buffer, i32 2 297 %s.buffer.2 = extractelement <8 x float> %s.buffer, i32 5 298 %s.buffer.3 = extractelement <8 x float> %s.buffer, i32 7 299 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %s.buffer.0, float %s.buffer.1, float %s.buffer.2, float %s.buffer.3, i1 true, i1 true) #0 300 ret void 301} 302 303; dwordx16 s.buffer.load 304; GCN-LABEL: {{^}}s_buffer_load_dwordx16: 305; VIGFX9_10: s_buffer_load_dwordx16 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]:[0-9]}}], 0x80 306; SICI: s_buffer_load_dwordx16 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]:[0-9]}}], 0x20 307define amdgpu_ps void @s_buffer_load_dwordx16(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in) #0 { 308main_body: 309 %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in 310 %s.buffer = call <16 x i32> @llvm.amdgcn.s.buffer.load.v16i32(<4 x i32> %tmp22, i32 128, i32 0) 311 %s.buffer.0 = extractelement <16 x i32> %s.buffer, i32 0 312 %s.buffer.0.float = bitcast i32 %s.buffer.0 to float 313 %s.buffer.1 = extractelement <16 x i32> %s.buffer, i32 3 314 %s.buffer.1.float = bitcast i32 %s.buffer.1 to float 315 %s.buffer.2 = extractelement <16 x i32> %s.buffer, i32 12 316 %s.buffer.2.float = bitcast i32 %s.buffer.2 to float 317 %s.buffer.3 = extractelement <16 x i32> %s.buffer, i32 15 318 %s.buffer.3.float = bitcast i32 %s.buffer.3 to float 319 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %s.buffer.0.float, float %s.buffer.1.float, float %s.buffer.2.float, float %s.buffer.3.float, i1 true, i1 true) #0 320 ret void 321} 322 323; GCN-LABEL: {{^}}s_buffer_load_dwordx16_v16f32: 324; VIGFX9_10: s_buffer_load_dwordx16 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]:[0-9]}}], 0x80 325; SICI: s_buffer_load_dwordx16 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]:[0-9]}}], 0x20 326define amdgpu_ps void @s_buffer_load_dwordx16_v16f32(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in) #0 { 327main_body: 328 %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in 329 %s.buffer = call <16 x float> @llvm.amdgcn.s.buffer.load.v16f32(<4 x i32> %tmp22, i32 128, i32 0) 330 %s.buffer.0 = extractelement <16 x float> %s.buffer, i32 0 331 %s.buffer.1 = extractelement <16 x float> %s.buffer, i32 3 332 %s.buffer.2 = extractelement <16 x float> %s.buffer, i32 12 333 %s.buffer.3 = extractelement <16 x float> %s.buffer, i32 15 334 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %s.buffer.0, float %s.buffer.1, float %s.buffer.2, float %s.buffer.3, i1 true, i1 true) #0 335 ret void 336} 337 338; GCN-LABEL: {{^}}smrd_sgpr_offset: 339; GCN: s_buffer_load_dword s{{[0-9]}}, s[0:3], s4 340define amdgpu_ps float @smrd_sgpr_offset(<4 x i32> inreg %desc, i32 inreg %offset) #0 { 341main_body: 342 %r = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %offset, i32 0) 343 ret float %r 344} 345 346; GCN-LABEL: {{^}}smrd_vgpr_offset: 347; GCN: buffer_load_dword v{{[0-9]}}, v0, s[0:3], 0 offen ; 348define amdgpu_ps float @smrd_vgpr_offset(<4 x i32> inreg %desc, i32 %offset) #0 { 349main_body: 350 %r = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %offset, i32 0) 351 ret float %r 352} 353 354; GCN-LABEL: {{^}}smrd_vgpr_offset_imm: 355; GCN-NEXT: %bb. 356; GCN-NEXT: buffer_load_dword v{{[0-9]}}, v0, s[0:3], 0 offen offset:4092 ; 357define amdgpu_ps float @smrd_vgpr_offset_imm(<4 x i32> inreg %desc, i32 %offset) #0 { 358main_body: 359 %off = add i32 %offset, 4092 360 %r = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %off, i32 0) 361 ret float %r 362} 363 364; GCN-LABEL: {{^}}smrd_vgpr_offset_imm_too_large: 365; GCN-NEXT: %bb. 366; SICI-NEXT: v_add_{{i|u}}32_e32 v0, {{(vcc, )?}}0x1000, v0 367; SICI-NEXT: buffer_load_dword v{{[0-9]}}, v0, s[0:3], 0 offen ; 368; VIGFX9_10-NEXT: buffer_load_dword v{{[0-9]}}, v0, s[0:3], 4 offen offset:4092 ; 369define amdgpu_ps float @smrd_vgpr_offset_imm_too_large(<4 x i32> inreg %desc, i32 %offset) #0 { 370main_body: 371 %off = add i32 %offset, 4096 372 %r = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %off, i32 0) 373 ret float %r 374} 375 376; GCN-LABEL: {{^}}smrd_imm_merged: 377; GCN-NEXT: %bb. 378; SICI-NEXT: s_buffer_load_dwordx4 s[{{[0-9]}}:{{[0-9]}}], s[0:3], 0x1 379; SICI-NEXT: s_buffer_load_dwordx2 s[{{[0-9]}}:{{[0-9]}}], s[0:3], 0x7 380; GFX10-NEXT: s_clause 381; VIGFX9_10-NEXT: s_buffer_load_dwordx4 s[{{[0-9]}}:{{[0-9]}}], s[0:3], 0x4 382; VIGFX9_10-NEXT: s_buffer_load_dwordx2 s[{{[0-9]}}:{{[0-9]}}], s[0:3], 0x1c 383define amdgpu_ps void @smrd_imm_merged(<4 x i32> inreg %desc) #0 { 384main_body: 385 %r1 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 4, i32 0) 386 %r2 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 8, i32 0) 387 %r3 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 12, i32 0) 388 %r4 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 16, i32 0) 389 %r5 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 28, i32 0) 390 %r6 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 32, i32 0) 391 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true) #0 392 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float undef, float undef, i1 true, i1 true) #0 393 ret void 394} 395 396; GCN-LABEL: {{^}}smrd_imm_merge_m0: 397; 398; GCN: s_buffer_load_dwordx2 399; SICIVI: s_mov_b32 m0 400; SICIVI_DAG: v_interp_p1_f32 401; SICIVI_DAG: v_interp_p1_f32 402; SICIVI_DAG: v_interp_p1_f32 403; SICIVI_DAG: v_interp_p2_f32 404; SICIVI_DAG: v_interp_p2_f32 405; SICIVI_DAG: v_interp_p2_f32 406; 407; extractelement does not result in movrels anymore for vectors gitting 8 dwords 408; SICIVI-NOT: s_mov_b32 m0 409; SICIVI-NOT: v_movrels_b32_e32 410; v_cndmask_b32_e32 411; v_cndmask_b32_e32 412; 413; Merging is still thwarted on GFX9 due to s_set_gpr_idx 414; 415define amdgpu_ps float @smrd_imm_merge_m0(<4 x i32> inreg %desc, i32 inreg %prim, float %u, float %v) #0 { 416main_body: 417 %idx1.f = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 0, i32 0) 418 %idx1 = bitcast float %idx1.f to i32 419 420 %v0.x1 = call nsz float @llvm.amdgcn.interp.p1(float %u, i32 0, i32 0, i32 %prim) 421 %v0.x = call nsz float @llvm.amdgcn.interp.p2(float %v0.x1, float %v, i32 0, i32 0, i32 %prim) 422 %v0.y1 = call nsz float @llvm.amdgcn.interp.p1(float %u, i32 0, i32 1, i32 %prim) 423 %v0.y = call nsz float @llvm.amdgcn.interp.p2(float %v0.y1, float %v, i32 0, i32 1, i32 %prim) 424 %v0.z1 = call nsz float @llvm.amdgcn.interp.p1(float %u, i32 0, i32 2, i32 %prim) 425 %v0.z = call nsz float @llvm.amdgcn.interp.p2(float %v0.z1, float %v, i32 0, i32 2, i32 %prim) 426 %v0.tmp0 = insertelement <3 x float> undef, float %v0.x, i32 0 427 %v0.tmp1 = insertelement <3 x float> %v0.tmp0, float %v0.y, i32 1 428 %v0 = insertelement <3 x float> %v0.tmp1, float %v0.z, i32 2 429 %a = extractelement <3 x float> %v0, i32 %idx1 430 431 %v1.x1 = call nsz float @llvm.amdgcn.interp.p1(float %u, i32 1, i32 0, i32 %prim) 432 %v1.x = call nsz float @llvm.amdgcn.interp.p2(float %v1.x1, float %v, i32 1, i32 0, i32 %prim) 433 %v1.y1 = call nsz float @llvm.amdgcn.interp.p1(float %u, i32 1, i32 1, i32 %prim) 434 %v1.y = call nsz float @llvm.amdgcn.interp.p2(float %v1.y1, float %v, i32 1, i32 1, i32 %prim) 435 %v1.z1 = call nsz float @llvm.amdgcn.interp.p1(float %u, i32 1, i32 2, i32 %prim) 436 %v1.z = call nsz float @llvm.amdgcn.interp.p2(float %v1.z1, float %v, i32 1, i32 2, i32 %prim) 437 %v1.tmp0 = insertelement <3 x float> undef, float %v0.x, i32 0 438 %v1.tmp1 = insertelement <3 x float> %v0.tmp0, float %v0.y, i32 1 439 %v1 = insertelement <3 x float> %v0.tmp1, float %v0.z, i32 2 440 441 %b = extractelement <3 x float> %v1, i32 %idx1 442 %c = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 4, i32 0) 443 444 %res.tmp = fadd float %a, %b 445 %res = fadd float %res.tmp, %c 446 ret float %res 447} 448 449; GCN-LABEL: {{^}}smrd_vgpr_merged: 450; GCN-NEXT: %bb. 451; GFX10-NEXT: s_clause 452; GCN-NEXT: buffer_load_dwordx4 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:4 453; GCN-NEXT: buffer_load_dwordx2 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:28 454define amdgpu_ps void @smrd_vgpr_merged(<4 x i32> inreg %desc, i32 %a) #0 { 455main_body: 456 %a1 = add i32 %a, 4 457 %a2 = add i32 %a, 8 458 %a3 = add i32 %a, 12 459 %a4 = add i32 %a, 16 460 %a5 = add i32 %a, 28 461 %a6 = add i32 %a, 32 462 %r1 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %a1, i32 0) 463 %r2 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %a2, i32 0) 464 %r3 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %a3, i32 0) 465 %r4 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %a4, i32 0) 466 %r5 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %a5, i32 0) 467 %r6 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %a6, i32 0) 468 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true) #0 469 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float undef, float undef, i1 true, i1 true) #0 470 ret void 471} 472 473; GCN-LABEL: {{^}}smrd_sgpr_descriptor_promoted 474; GCN: v_readfirstlane 475define amdgpu_cs void @smrd_sgpr_descriptor_promoted([0 x i8] addrspace(4)* inreg noalias dereferenceable(18446744073709551615), i32) #0 { 476main_body: 477 %descptr = bitcast [0 x i8] addrspace(4)* %0 to <4 x i32> addrspace(4)*, !amdgpu.uniform !0 478 br label %.outer_loop_header 479 480ret_block: ; preds = %.outer, %.label22, %main_body 481 ret void 482 483.outer_loop_header: 484 br label %.inner_loop_header 485 486.inner_loop_header: ; preds = %.inner_loop_body, %.outer_loop_header 487 %loopctr.1 = phi i32 [ 0, %.outer_loop_header ], [ %loopctr.2, %.inner_loop_body ] 488 %loopctr.2 = add i32 %loopctr.1, 1 489 %inner_br1 = icmp slt i32 %loopctr.2, 10 490 br i1 %inner_br1, label %.inner_loop_body, label %ret_block 491 492.inner_loop_body: 493 %descriptor = load <4 x i32>, <4 x i32> addrspace(4)* %descptr, align 16, !invariant.load !0 494 %load1result = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %descriptor, i32 0, i32 0) 495 store float %load1result, float addrspace(1)* undef 496 %inner_br2 = icmp uge i32 %1, 10 497 br i1 %inner_br2, label %.inner_loop_header, label %.outer_loop_body 498 499.outer_loop_body: 500 %offset = shl i32 %loopctr.2, 6 501 %load2result = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %descriptor, i32 %offset, i32 0) 502 %outer_br = fcmp ueq float %load2result, 0x0 503 br i1 %outer_br, label %.outer_loop_header, label %ret_block 504} 505 506; SMRD load with a non-const offset 507; GCN-LABEL: {{^}}smrd_load_nonconst0: 508; SIVIGFX9_10: s_buffer_load_dword s{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} 509; SIVIGFX9_10: s_buffer_load_dword s{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} 510; CI: s_buffer_load_dword s{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} 511; CI: s_buffer_load_dword s{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} 512; GCN: s_endpgm 513define amdgpu_ps void @smrd_load_nonconst0(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in, i32 inreg %ncoff) #0 { 514main_body: 515 %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg, i32 0 516 %tmp20 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp 517 %tmp21 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 %ncoff, i32 0) 518 %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in 519 %s.buffer = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %tmp22, i32 %ncoff, i32 0) 520 %s.buffer.float = bitcast i32 %s.buffer to float 521 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %tmp21, float %tmp21, float %tmp21, float %s.buffer.float, i1 true, i1 true) #0 522 ret void 523} 524 525; SMRD load with a non-const non-uniform offset 526; GCN-LABEL: {{^}}smrd_load_nonconst1: 527; SIVIGFX9_10: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen 528; SIVIGFX9_10: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen 529; CI: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen 530; CI: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen 531; GCN: s_endpgm 532define amdgpu_ps void @smrd_load_nonconst1(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in, i32 %ncoff) #0 { 533main_body: 534 %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg, i32 0 535 %tmp20 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp 536 %tmp21 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 %ncoff, i32 0) 537 %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in 538 %s.buffer = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %tmp22, i32 %ncoff, i32 0) 539 %s.buffer.float = bitcast i32 %s.buffer to float 540 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %tmp21, float %tmp21, float %tmp21, float %s.buffer.float, i1 true, i1 true) #0 541 ret void 542} 543 544; SMRD load with a non-const non-uniform offset of > 4 dwords (requires splitting) 545; GCN-LABEL: {{^}}smrd_load_nonconst2: 546; SIVIGFX9_10-DAG: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen 547; SIVIGFX9_10-DAG: buffer_load_dwordx4 v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen 548; CI: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen 549; CI: buffer_load_dwordx4 v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen 550; GCN: s_endpgm 551define amdgpu_ps void @smrd_load_nonconst2(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in, i32 %ncoff) #0 { 552main_body: 553 %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg, i32 0 554 %tmp20 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp 555 %tmp21 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 %ncoff, i32 0) 556 %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in 557 %s.buffer = call <8 x i32> @llvm.amdgcn.s.buffer.load.v8i32(<4 x i32> %tmp22, i32 %ncoff, i32 0) 558 %s.buffer.elt = extractelement <8 x i32> %s.buffer, i32 1 559 %s.buffer.float = bitcast i32 %s.buffer.elt to float 560 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %tmp21, float %tmp21, float %tmp21, float %s.buffer.float, i1 true, i1 true) #0 561 ret void 562} 563 564; SMRD load with a non-const non-uniform offset of > 4 dwords (requires splitting) 565; GCN-LABEL: {{^}}smrd_load_nonconst3: 566; GCN-DAG: buffer_load_dwordx4 v[0:3], v{{[0-9]+}}, s[0:3], 0 offen ; 567; GCN-DAG: buffer_load_dwordx4 v[4:7], v{{[0-9]+}}, s[0:3], 0 offen offset:16 ; 568; GCN-DAG: buffer_load_dwordx4 v[8:11], v{{[0-9]+}}, s[0:3], 0 offen offset:32 ; 569; GCN-DAG: buffer_load_dwordx4 v[12:15], v{{[0-9]+}}, s[0:3], 0 offen offset:48 ; 570; GCN: ; return to shader part epilog 571define amdgpu_ps <16 x float> @smrd_load_nonconst3(<4 x i32> inreg %rsrc, i32 %off) #0 { 572main_body: 573 %ld = call <16 x i32> @llvm.amdgcn.s.buffer.load.v16i32(<4 x i32> %rsrc, i32 %off, i32 0) 574 %bc = bitcast <16 x i32> %ld to <16 x float> 575 ret <16 x float> %bc 576} 577 578; GCN-LABEL: {{^}}smrd_load_nonconst4: 579; SICI: v_add_i32_e32 v{{[0-9]+}}, vcc, 0xff8, v0 ; 580; SICI-DAG: buffer_load_dwordx4 v[0:3], v{{[0-9]+}}, s[0:3], 0 offen ; 581; SICI-DAG: buffer_load_dwordx4 v[4:7], v{{[0-9]+}}, s[0:3], 0 offen offset:16 ; 582; SICI-DAG: buffer_load_dwordx4 v[8:11], v{{[0-9]+}}, s[0:3], 0 offen offset:32 ; 583; SICI-DAG: buffer_load_dwordx4 v[12:15], v{{[0-9]+}}, s[0:3], 0 offen offset:48 ; 584; VIGFX9_10-DAG: buffer_load_dwordx4 v[0:3], v{{[0-9]+}}, s[0:3], 56 offen offset:4032 ; 585; VIGFX9_10-DAG: buffer_load_dwordx4 v[4:7], v{{[0-9]+}}, s[0:3], 56 offen offset:4048 ; 586; VIGFX9_10-DAG: buffer_load_dwordx4 v[8:11], v{{[0-9]+}}, s[0:3], 56 offen offset:4064 ; 587; VIGFX9_10-DAG: buffer_load_dwordx4 v[12:15], v{{[0-9]+}}, s[0:3], 56 offen offset:4080 ; 588; GCN: ; return to shader part epilog 589define amdgpu_ps <16 x float> @smrd_load_nonconst4(<4 x i32> inreg %rsrc, i32 %off) #0 { 590main_body: 591 %off.2 = add i32 %off, 4088 592 %ld = call <16 x i32> @llvm.amdgcn.s.buffer.load.v16i32(<4 x i32> %rsrc, i32 %off.2, i32 0) 593 %bc = bitcast <16 x i32> %ld to <16 x float> 594 ret <16 x float> %bc 595} 596 597; GCN-LABEL: {{^}}smrd_load_nonconst5: 598; SICI: v_add_i32_e32 v{{[0-9]+}}, vcc, 0x1004, v0 599; SICI-DAG: buffer_load_dwordx4 v[0:3], v{{[0-9]+}}, s[0:3], 0 offen ; 600; SICI-DAG: buffer_load_dwordx4 v[4:7], v{{[0-9]+}}, s[0:3], 0 offen offset:16 ; 601; SICI-DAG: buffer_load_dwordx4 v[8:11], v{{[0-9]+}}, s[0:3], 0 offen offset:32 ; 602; SICI-DAG: buffer_load_dwordx4 v[12:15], v{{[0-9]+}}, s[0:3], 0 offen offset:48 ; 603; VIGFX9_10: s_movk_i32 s4, 0xfc0 604; VIGFX9_10-DAG: buffer_load_dwordx4 v[0:3], v{{[0-9]+}}, s[0:3], s4 offen offset:68 ; 605; VIGFX9_10-DAG: buffer_load_dwordx4 v[4:7], v{{[0-9]+}}, s[0:3], s4 offen offset:84 ; 606; VIGFX9_10-DAG: buffer_load_dwordx4 v[8:11], v{{[0-9]+}}, s[0:3], s4 offen offset:100 ; 607; VIGFX9_10-DAG: buffer_load_dwordx4 v[12:15], v{{[0-9]+}}, s[0:3], s4 offen offset:116 ; 608; GCN: ; return to shader part epilog 609define amdgpu_ps <16 x float> @smrd_load_nonconst5(<4 x i32> inreg %rsrc, i32 %off) #0 { 610main_body: 611 %off.2 = add i32 %off, 4100 612 %ld = call <16 x i32> @llvm.amdgcn.s.buffer.load.v16i32(<4 x i32> %rsrc, i32 %off.2, i32 0) 613 %bc = bitcast <16 x i32> %ld to <16 x float> 614 ret <16 x float> %bc 615} 616 617; SMRD load dwordx2 618; GCN-LABEL: {{^}}smrd_load_dwordx2: 619; SIVIGFX9_10: s_buffer_load_dwordx2 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} 620; CI: s_buffer_load_dwordx2 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} 621; GCN: s_endpgm 622define amdgpu_ps void @smrd_load_dwordx2(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in, i32 inreg %ncoff) #0 { 623main_body: 624 %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in 625 %s.buffer = call <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32> %tmp22, i32 %ncoff, i32 0) 626 %s.buffer.float = bitcast <2 x i32> %s.buffer to <2 x float> 627 %r.1 = extractelement <2 x float> %s.buffer.float, i32 0 628 %r.2 = extractelement <2 x float> %s.buffer.float, i32 1 629 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r.1, float %r.1, float %r.1, float %r.2, i1 true, i1 true) #0 630 ret void 631} 632 633; GCN-LABEL: {{^}}smrd_uniform_loop: 634; 635; TODO: we should keep the loop counter in an SGPR 636; 637; GCN: s_buffer_load_dword 638define amdgpu_ps float @smrd_uniform_loop(<4 x i32> inreg %desc, i32 %bound) #0 { 639main_body: 640 br label %loop 641 642loop: 643 %counter = phi i32 [ 0, %main_body ], [ %counter.next, %loop ] 644 %sum = phi float [ 0.0, %main_body ], [ %sum.next, %loop ] 645 %offset = shl i32 %counter, 2 646 %v = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %offset, i32 0) 647 %sum.next = fadd float %sum, %v 648 %counter.next = add i32 %counter, 1 649 %cc = icmp uge i32 %counter.next, %bound 650 br i1 %cc, label %exit, label %loop 651 652exit: 653 ret float %sum.next 654} 655 656 657; GCN-LABEL: {{^}}smrd_uniform_loop2: 658; (this test differs from smrd_uniform_loop by the more complex structure of phis, 659; which used to confuse the DivergenceAnalysis after structurization) 660; 661; TODO: we should keep the loop counter in an SGPR and use an S_BUFFER_LOAD 662; 663; GCN: buffer_load_dword 664define amdgpu_ps float @smrd_uniform_loop2(<4 x i32> inreg %desc, i32 %bound, i32 %bound.a) #0 { 665main_body: 666 br label %loop 667 668loop: 669 %counter = phi i32 [ 0, %main_body ], [ %counter.next, %loop.a ], [ %counter.next, %loop.b ] 670 %sum = phi float [ 0.0, %main_body ], [ %sum.next, %loop.a ], [ %sum.next.b, %loop.b ] 671 %offset = shl i32 %counter, 2 672 %v = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %offset, i32 0) 673 %sum.next = fadd float %sum, %v 674 %counter.next = add i32 %counter, 1 675 %cc = icmp uge i32 %counter.next, %bound 676 br i1 %cc, label %exit, label %loop.a 677 678loop.a: 679 %cc.a = icmp uge i32 %counter.next, %bound.a 680 br i1 %cc, label %loop, label %loop.b 681 682loop.b: 683 %sum.next.b = fadd float %sum.next, 1.0 684 br label %loop 685 686exit: 687 ret float %sum.next 688} 689 690; This test checks that the load after some control flow with an offset based 691; on a divergent shader input is correctly recognized as divergent. This was 692; reduced from an actual regression. Yes, the %unused argument matters, as 693; well as the fact that %arg4 is a vector. 694; 695; GCN-LABEL: {{^}}arg_divergence: 696; GCN: buffer_load_dword v0, v0, 697; GCN-NEXT: s_waitcnt 698; GCN-NEXT: ; return to shader part epilog 699define amdgpu_cs float @arg_divergence(i32 inreg %unused, <3 x i32> %arg4) #0 { 700main_body: 701 br i1 undef, label %if1, label %endif1 702 703if1: ; preds = %main_body 704 store i32 0, i32 addrspace(3)* undef, align 4 705 br label %endif1 706 707endif1: ; preds = %if1, %main_body 708 %tmp13 = extractelement <3 x i32> %arg4, i32 0 709 %tmp97 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 %tmp13, i32 0) 710 ret float %tmp97 711} 712 713; GCN-LABEL: {{^}}s_buffer_load_f32: 714; GCN: s_buffer_load_dword s0, s[0:3], s4 715define amdgpu_ps void @s_buffer_load_f32(<4 x i32> inreg %rsrc, i32 inreg %offset) { 716 %sgpr = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %rsrc, i32 %offset, i32 0) 717 call void asm sideeffect "; use $0", "s"(float %sgpr) 718 ret void 719} 720 721; GCN-LABEL: {{^}}s_buffer_load_v2f32: 722; GCN: s_buffer_load_dwordx2 s[0:1], s[0:3], s4 723define amdgpu_ps void @s_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 inreg %offset) { 724 %sgpr = call <2 x float> @llvm.amdgcn.s.buffer.load.v2f32(<4 x i32> %rsrc, i32 %offset, i32 0) 725 call void asm sideeffect "; use $0", "s"(<2 x float> %sgpr) 726 ret void 727} 728 729; GCN-LABEL: {{^}}s_buffer_load_v4f32: 730; GCN: s_buffer_load_dwordx4 s[0:3], s[0:3], s4 731define amdgpu_ps void @s_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 inreg %offset) { 732 %sgpr = call <4 x float> @llvm.amdgcn.s.buffer.load.v4f32(<4 x i32> %rsrc, i32 %offset, i32 0) 733 call void asm sideeffect "; use $0", "s"(<4 x float> %sgpr) 734 ret void 735} 736 737; GCN-LABEL: {{^}}s_buffer_load_v8f32: 738; GCN: s_buffer_load_dwordx8 s[0:7], s[0:3], s4 739define amdgpu_ps void @s_buffer_load_v8f32(<4 x i32> inreg %rsrc, i32 inreg %offset) { 740 %sgpr = call <8 x float> @llvm.amdgcn.s.buffer.load.v8f32(<4 x i32> %rsrc, i32 %offset, i32 0) 741 call void asm sideeffect "; use $0", "s"(<8 x float> %sgpr) 742 ret void 743} 744 745; GCN-LABEL: {{^}}s_buffer_load_v16f32: 746; GCN: s_buffer_load_dwordx16 s[0:15], s[0:3], s4 747define amdgpu_ps void @s_buffer_load_v16f32(<4 x i32> inreg %rsrc, i32 inreg %offset) { 748 %sgpr = call <16 x float> @llvm.amdgcn.s.buffer.load.v16f32(<4 x i32> %rsrc, i32 %offset, i32 0) 749 call void asm sideeffect "; use $0", "s"(<16 x float> %sgpr) 750 ret void 751} 752 753declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #0 754declare float @llvm.amdgcn.interp.p1(float, i32, i32, i32) #2 755declare float @llvm.amdgcn.interp.p2(float, float, i32, i32, i32) #2 756 757declare i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32>, i32, i32) #1 758declare <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32>, i32, i32) 759declare <4 x i32> @llvm.amdgcn.s.buffer.load.v4i32(<4 x i32>, i32, i32) 760declare <8 x i32> @llvm.amdgcn.s.buffer.load.v8i32(<4 x i32>, i32, i32) 761declare <16 x i32> @llvm.amdgcn.s.buffer.load.v16i32(<4 x i32>, i32, i32) 762 763declare float @llvm.amdgcn.s.buffer.load.f32(<4 x i32>, i32, i32) 764declare <2 x float> @llvm.amdgcn.s.buffer.load.v2f32(<4 x i32>, i32, i32) 765declare <4 x float> @llvm.amdgcn.s.buffer.load.v4f32(<4 x i32>, i32, i32) 766declare <8 x float> @llvm.amdgcn.s.buffer.load.v8f32(<4 x i32>, i32, i32) 767declare <16 x float> @llvm.amdgcn.s.buffer.load.v16f32(<4 x i32>, i32, i32) 768 769attributes #0 = { nounwind } 770attributes #1 = { nounwind readnone } 771attributes #2 = { nounwind readnone speculatable } 772 773!0 = !{} 774