1; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=SICIVI -check-prefix=SICI -check-prefix=SIVIGFX9_10 %s 2; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=CI -check-prefix=GCN -check-prefix=SICIVI -check-prefix=SICI %s 3; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=VI -check-prefix=GCN -check-prefix=SICIVI -check-prefix=VIGFX9_10 -check-prefix=SIVIGFX9_10 %s 4; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=GFX9 -check-prefix=GCN -check-prefix=VIGFX9_10 -check-prefix=SIVIGFX9_10 %s 5; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=GFX10 -check-prefix=GCN -check-prefix=VIGFX9_10 -check-prefix=SIVIGFX9_10 %s 6 7; SMRD load with an immediate offset. 8; GCN-LABEL: {{^}}smrd0: 9; SICI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x1 ; encoding: [0x01 10; VIGFX9_10: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x4 11define amdgpu_kernel void @smrd0(i32 addrspace(1)* %out, i32 addrspace(4)* %ptr) #0 { 12entry: 13 %tmp = getelementptr i32, i32 addrspace(4)* %ptr, i64 1 14 %tmp1 = load i32, i32 addrspace(4)* %tmp 15 store i32 %tmp1, i32 addrspace(1)* %out 16 ret void 17} 18 19; SMRD load with the largest possible immediate offset. 20; GCN-LABEL: {{^}}smrd1: 21; SICI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0xff ; encoding: [0xff,0x{{[0-9]+[137]}} 22; VIGFX9_10: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x3fc 23define amdgpu_kernel void @smrd1(i32 addrspace(1)* %out, i32 addrspace(4)* %ptr) #0 { 24entry: 25 %tmp = getelementptr i32, i32 addrspace(4)* %ptr, i64 255 26 %tmp1 = load i32, i32 addrspace(4)* %tmp 27 store i32 %tmp1, i32 addrspace(1)* %out 28 ret void 29} 30 31; SMRD load with an offset greater than the largest possible immediate. 32; GCN-LABEL: {{^}}smrd2: 33; SI: s_movk_i32 s[[OFFSET:[0-9]]], 0x400 34; SI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], s[[OFFSET]] ; encoding: [0x0[[OFFSET]] 35; CI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x100 36; VIGFX9_10: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x400 37; GCN: s_endpgm 38define amdgpu_kernel void @smrd2(i32 addrspace(1)* %out, i32 addrspace(4)* %ptr) #0 { 39entry: 40 %tmp = getelementptr i32, i32 addrspace(4)* %ptr, i64 256 41 %tmp1 = load i32, i32 addrspace(4)* %tmp 42 store i32 %tmp1, i32 addrspace(1)* %out 43 ret void 44} 45 46; SMRD load with a 64-bit offset 47; GCN-LABEL: {{^}}smrd3: 48; FIXME: There are too many copies here because we don't fold immediates 49; through REG_SEQUENCE 50; SI: s_load_dwordx2 s[{{[0-9]:[0-9]}}], s[{{[0-9]:[0-9]}}], 0x13 ; encoding: [0x13 51; TODO: Add VI checks 52; GCN: s_endpgm 53define amdgpu_kernel void @smrd3(i32 addrspace(1)* %out, [8 x i32], i32 addrspace(4)* %ptr) #0 { 54entry: 55 %tmp = getelementptr i32, i32 addrspace(4)* %ptr, i64 4294967296 56 %tmp1 = load i32, i32 addrspace(4)* %tmp 57 store i32 %tmp1, i32 addrspace(1)* %out 58 ret void 59} 60 61; SMRD load with the largest possible immediate offset on VI 62; GCN-LABEL: {{^}}smrd4: 63; SI: s_mov_b32 [[OFFSET:s[0-9]+]], 0xffffc 64; SI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], [[OFFSET]] 65; CI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x3ffff 66; VIGFX9_10: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0xffffc 67define amdgpu_kernel void @smrd4(i32 addrspace(1)* %out, i32 addrspace(4)* %ptr) #0 { 68entry: 69 %tmp = getelementptr i32, i32 addrspace(4)* %ptr, i64 262143 70 %tmp1 = load i32, i32 addrspace(4)* %tmp 71 store i32 %tmp1, i32 addrspace(1)* %out 72 ret void 73} 74 75; SMRD load with an offset greater than the largest possible immediate on VI 76; GCN-LABEL: {{^}}smrd5: 77; SIVIGFX9_10: s_mov_b32 [[OFFSET:s[0-9]+]], 0x100000 78; SIVIGFX9_10: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], [[OFFSET]] 79; CI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x40000 80; GCN: s_endpgm 81define amdgpu_kernel void @smrd5(i32 addrspace(1)* %out, i32 addrspace(4)* %ptr) #0 { 82entry: 83 %tmp = getelementptr i32, i32 addrspace(4)* %ptr, i64 262144 84 %tmp1 = load i32, i32 addrspace(4)* %tmp 85 store i32 %tmp1, i32 addrspace(1)* %out 86 ret void 87} 88 89; GCN-LABEL: {{^}}smrd_hazard: 90; GCN-DAG: s_mov_b32 s3, 3 91; GCN-DAG: s_mov_b32 s2, 2 92; GCN-DAG: s_mov_b32 s1, 1 93; GCN-DAG: s_mov_b32 s0, 0 94; SI-NEXT: nop 3 95; GFX10-NEXT: ; implicit-def: $vcc_hi 96; GCN-NEXT: s_buffer_load_dword s0, s[0:3], 0x0 97define amdgpu_ps float @smrd_hazard(<4 x i32> inreg %desc) #0 { 98main_body: 99 %d0 = insertelement <4 x i32> undef, i32 0, i32 0 100 %d1 = insertelement <4 x i32> %d0, i32 1, i32 1 101 %d2 = insertelement <4 x i32> %d1, i32 2, i32 2 102 %d3 = insertelement <4 x i32> %d2, i32 3, i32 3 103 %r = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %d3, i32 0, i32 0) 104 ret float %r 105} 106 107; SMRD load using the load.const.v4i32 intrinsic with an immediate offset 108; GCN-LABEL: {{^}}smrd_load_const0: 109; SICI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x4 ; encoding: [0x04 110; VIGFX9_10: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x10 111define amdgpu_ps void @smrd_load_const0(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in) #0 { 112main_body: 113 %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg, i32 0 114 %tmp20 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp 115 %tmp21 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 16, i32 0) 116 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %tmp21, float %tmp21, float %tmp21, float %tmp21, i1 true, i1 true) #0 117 ret void 118} 119 120; SMRD load using the load.const.v4i32 intrinsic with the largest possible immediate 121; offset. 122; GCN-LABEL: {{^}}smrd_load_const1: 123; SICI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0xff ; encoding: [0xff 124; SICI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0xff glc ; encoding: [0xff 125; VIGFX9_10-DAG: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x3fc ; 126; VIGFX9_10-DAG: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x3fc glc ; 127define amdgpu_ps void @smrd_load_const1(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in) #0 { 128main_body: 129 %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg, i32 0 130 %tmp20 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp 131 %tmp21 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 1020, i32 0) 132 %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in 133 %s.buffer = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %tmp22, i32 1020, i32 1) 134 %s.buffer.float = bitcast i32 %s.buffer to float 135 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %tmp21, float %tmp21, float %tmp21, float %s.buffer.float, i1 true, i1 true) #0 136 ret void 137} 138 139; SMRD load using the load.const.v4i32 intrinsic with an offset greater than the 140; largets possible immediate. 141; immediate offset. 142; GCN-LABEL: {{^}}smrd_load_const2: 143; SI: s_movk_i32 s[[OFFSET:[0-9]]], 0x400 144; SI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], s[[OFFSET]] ; encoding: [0x0[[OFFSET]] 145; SI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], s[[OFFSET]] ; encoding: [0x0[[OFFSET]] 146; CI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x100 147; CI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x100 148; VIGFX9_10: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x400 149; VIGFX9_10: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x400 150define amdgpu_ps void @smrd_load_const2(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in) #0 { 151main_body: 152 %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg, i32 0 153 %tmp20 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp 154 %tmp21 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 1024, i32 0) 155 %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in 156 %s.buffer = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %tmp22, i32 1024, i32 0) 157 %s.buffer.float = bitcast i32 %s.buffer to float 158 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %tmp21, float %tmp21, float %tmp21, float %s.buffer.float, i1 true, i1 true) #0 159 ret void 160} 161 162; SMRD load with the largest possible immediate offset on VI 163; GCN-LABEL: {{^}}smrd_load_const3: 164; SI: s_mov_b32 [[OFFSET:s[0-9]+]], 0xffffc 165; SI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], [[OFFSET]] 166; SI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], [[OFFSET]] 167; CI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x3ffff 168; CI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x3ffff 169; VIGFX9_10: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0xffffc 170; VIGFX9_10: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0xffffc 171define amdgpu_ps void @smrd_load_const3(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in) #0 { 172main_body: 173 %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg, i32 0 174 %tmp20 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp 175 %tmp21 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 1048572, i32 0) 176 %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in 177 %s.buffer = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %tmp22, i32 1048572, i32 0) 178 %s.buffer.float = bitcast i32 %s.buffer to float 179 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %tmp21, float %tmp21, float %tmp21, float %s.buffer.float, i1 true, i1 true) #0 180 ret void 181} 182 183; SMRD load with an offset greater than the largest possible immediate on VI 184; GCN-LABEL: {{^}}smrd_load_const4: 185; SIVIGFX9_10: s_mov_b32 [[OFFSET:s[0-9]+]], 0x100000 186; SIVIGFX9_10: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], [[OFFSET]] 187; SIVIGFX9_10: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], [[OFFSET]] 188; CI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x40000 189; CI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x40000 190; GCN: s_endpgm 191define amdgpu_ps void @smrd_load_const4(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in) #0 { 192main_body: 193 %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg, i32 0 194 %tmp20 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp 195 %tmp21 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 1048576, i32 0) 196 %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in 197 %s.buffer = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %tmp22, i32 1048576, i32 0) 198 %s.buffer.float = bitcast i32 %s.buffer to float 199 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %tmp21, float %tmp21, float %tmp21, float %s.buffer.float, i1 true, i1 true) #0 200 ret void 201} 202 203; dwordx2 s.buffer.load 204; GCN-LABEL: {{^}}s_buffer_load_dwordx2: 205; VIGFX9_10: s_buffer_load_dwordx2 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]:[0-9]}}], 0x80 206; SICI: s_buffer_load_dwordx2 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]:[0-9]}}], 0x20 207define amdgpu_ps void @s_buffer_load_dwordx2(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in) #0 { 208main_body: 209 %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in 210 %s.buffer = call <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32> %tmp22, i32 128, i32 0) 211 %s.buffer.0 = extractelement <2 x i32> %s.buffer, i32 0 212 %s.buffer.0.float = bitcast i32 %s.buffer.0 to float 213 %s.buffer.1 = extractelement <2 x i32> %s.buffer, i32 1 214 %s.buffer.1.float = bitcast i32 %s.buffer.1 to float 215 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %s.buffer.0.float, float %s.buffer.1.float, float %s.buffer.0.float, float %s.buffer.1.float, i1 true, i1 true) #0 216 ret void 217} 218 219; dwordx4 s.buffer.load 220; GCN-LABEL: {{^}}s_buffer_load_dwordx4: 221; VIGFX9_10: s_buffer_load_dwordx4 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]:[0-9]}}], 0x80 222; SICI: s_buffer_load_dwordx4 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]:[0-9]}}], 0x20 223define amdgpu_ps void @s_buffer_load_dwordx4(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in) #0 { 224main_body: 225 %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in 226 %s.buffer = call <4 x i32> @llvm.amdgcn.s.buffer.load.v4i32(<4 x i32> %tmp22, i32 128, i32 0) 227 %s.buffer.0 = extractelement <4 x i32> %s.buffer, i32 0 228 %s.buffer.0.float = bitcast i32 %s.buffer.0 to float 229 %s.buffer.1 = extractelement <4 x i32> %s.buffer, i32 1 230 %s.buffer.1.float = bitcast i32 %s.buffer.1 to float 231 %s.buffer.2 = extractelement <4 x i32> %s.buffer, i32 2 232 %s.buffer.2.float = bitcast i32 %s.buffer.2 to float 233 %s.buffer.3 = extractelement <4 x i32> %s.buffer, i32 3 234 %s.buffer.3.float = bitcast i32 %s.buffer.3 to float 235 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %s.buffer.0.float, float %s.buffer.1.float, float %s.buffer.2.float, float %s.buffer.3.float, i1 true, i1 true) #0 236 ret void 237} 238 239; dwordx8 s.buffer.load 240; GCN-LABEL: {{^}}s_buffer_load_dwordx8: 241; VIGFX9_10: s_buffer_load_dwordx8 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]:[0-9]}}], 0x80 242; SICI: s_buffer_load_dwordx8 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]:[0-9]}}], 0x20 243define amdgpu_ps void @s_buffer_load_dwordx8(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in) #0 { 244main_body: 245 %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in 246 %s.buffer = call <8 x i32> @llvm.amdgcn.s.buffer.load.v8i32(<4 x i32> %tmp22, i32 128, i32 0) 247 %s.buffer.0 = extractelement <8 x i32> %s.buffer, i32 0 248 %s.buffer.0.float = bitcast i32 %s.buffer.0 to float 249 %s.buffer.1 = extractelement <8 x i32> %s.buffer, i32 2 250 %s.buffer.1.float = bitcast i32 %s.buffer.1 to float 251 %s.buffer.2 = extractelement <8 x i32> %s.buffer, i32 5 252 %s.buffer.2.float = bitcast i32 %s.buffer.2 to float 253 %s.buffer.3 = extractelement <8 x i32> %s.buffer, i32 7 254 %s.buffer.3.float = bitcast i32 %s.buffer.3 to float 255 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %s.buffer.0.float, float %s.buffer.1.float, float %s.buffer.2.float, float %s.buffer.3.float, i1 true, i1 true) #0 256 ret void 257} 258 259; dwordx8 s.buffer.load 260; GCN-LABEL: {{^}}s_buffer_load_dwordx8_v8f32: 261; VIGFX9_10: s_buffer_load_dwordx8 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]:[0-9]}}], 0x80 262; SICI: s_buffer_load_dwordx8 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]:[0-9]}}], 0x20 263define amdgpu_ps void @s_buffer_load_dwordx8_v8f32(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in) #0 { 264main_body: 265 %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in 266 %s.buffer = call <8 x float> @llvm.amdgcn.s.buffer.load.v8f32(<4 x i32> %tmp22, i32 128, i32 0) 267 %s.buffer.0 = extractelement <8 x float> %s.buffer, i32 0 268 %s.buffer.1 = extractelement <8 x float> %s.buffer, i32 2 269 %s.buffer.2 = extractelement <8 x float> %s.buffer, i32 5 270 %s.buffer.3 = extractelement <8 x float> %s.buffer, i32 7 271 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %s.buffer.0, float %s.buffer.1, float %s.buffer.2, float %s.buffer.3, i1 true, i1 true) #0 272 ret void 273} 274 275; dwordx16 s.buffer.load 276; GCN-LABEL: {{^}}s_buffer_load_dwordx16: 277; VIGFX9_10: s_buffer_load_dwordx16 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]:[0-9]}}], 0x80 278; SICI: s_buffer_load_dwordx16 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]:[0-9]}}], 0x20 279define amdgpu_ps void @s_buffer_load_dwordx16(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in) #0 { 280main_body: 281 %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in 282 %s.buffer = call <16 x i32> @llvm.amdgcn.s.buffer.load.v16i32(<4 x i32> %tmp22, i32 128, i32 0) 283 %s.buffer.0 = extractelement <16 x i32> %s.buffer, i32 0 284 %s.buffer.0.float = bitcast i32 %s.buffer.0 to float 285 %s.buffer.1 = extractelement <16 x i32> %s.buffer, i32 3 286 %s.buffer.1.float = bitcast i32 %s.buffer.1 to float 287 %s.buffer.2 = extractelement <16 x i32> %s.buffer, i32 12 288 %s.buffer.2.float = bitcast i32 %s.buffer.2 to float 289 %s.buffer.3 = extractelement <16 x i32> %s.buffer, i32 15 290 %s.buffer.3.float = bitcast i32 %s.buffer.3 to float 291 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %s.buffer.0.float, float %s.buffer.1.float, float %s.buffer.2.float, float %s.buffer.3.float, i1 true, i1 true) #0 292 ret void 293} 294 295; GCN-LABEL: {{^}}s_buffer_load_dwordx16_v16f32: 296; VIGFX9_10: s_buffer_load_dwordx16 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]:[0-9]}}], 0x80 297; SICI: s_buffer_load_dwordx16 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]:[0-9]}}], 0x20 298define amdgpu_ps void @s_buffer_load_dwordx16_v16f32(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in) #0 { 299main_body: 300 %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in 301 %s.buffer = call <16 x float> @llvm.amdgcn.s.buffer.load.v16f32(<4 x i32> %tmp22, i32 128, i32 0) 302 %s.buffer.0 = extractelement <16 x float> %s.buffer, i32 0 303 %s.buffer.1 = extractelement <16 x float> %s.buffer, i32 3 304 %s.buffer.2 = extractelement <16 x float> %s.buffer, i32 12 305 %s.buffer.3 = extractelement <16 x float> %s.buffer, i32 15 306 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %s.buffer.0, float %s.buffer.1, float %s.buffer.2, float %s.buffer.3, i1 true, i1 true) #0 307 ret void 308} 309 310; GCN-LABEL: {{^}}smrd_sgpr_offset: 311; GCN: s_buffer_load_dword s{{[0-9]}}, s[0:3], s4 312define amdgpu_ps float @smrd_sgpr_offset(<4 x i32> inreg %desc, i32 inreg %offset) #0 { 313main_body: 314 %r = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %offset, i32 0) 315 ret float %r 316} 317 318; GCN-LABEL: {{^}}smrd_vgpr_offset: 319; GCN: buffer_load_dword v{{[0-9]}}, v0, s[0:3], 0 offen ; 320define amdgpu_ps float @smrd_vgpr_offset(<4 x i32> inreg %desc, i32 %offset) #0 { 321main_body: 322 %r = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %offset, i32 0) 323 ret float %r 324} 325 326; GCN-LABEL: {{^}}smrd_vgpr_offset_imm: 327; GCN-NEXT: %bb. 328; GCN-NEXT: buffer_load_dword v{{[0-9]}}, v0, s[0:3], 0 offen offset:4092 ; 329define amdgpu_ps float @smrd_vgpr_offset_imm(<4 x i32> inreg %desc, i32 %offset) #0 { 330main_body: 331 %off = add i32 %offset, 4092 332 %r = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %off, i32 0) 333 ret float %r 334} 335 336; GCN-LABEL: {{^}}smrd_vgpr_offset_imm_too_large: 337; GCN-NEXT: %bb. 338; SICI-NEXT: v_add_{{i|u}}32_e32 v0, {{(vcc, )?}}0x1000, v0 339; SICI-NEXT: buffer_load_dword v{{[0-9]}}, v0, s[0:3], 0 offen ; 340; VIGFX9_10-NEXT: buffer_load_dword v{{[0-9]}}, v0, s[0:3], 4 offen offset:4092 ; 341define amdgpu_ps float @smrd_vgpr_offset_imm_too_large(<4 x i32> inreg %desc, i32 %offset) #0 { 342main_body: 343 %off = add i32 %offset, 4096 344 %r = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %off, i32 0) 345 ret float %r 346} 347 348; GCN-LABEL: {{^}}smrd_imm_merged: 349; GCN-NEXT: %bb. 350; SICI-NEXT: s_buffer_load_dwordx4 s[{{[0-9]}}:{{[0-9]}}], s[0:3], 0x1 351; SICI-NEXT: s_buffer_load_dwordx2 s[{{[0-9]}}:{{[0-9]}}], s[0:3], 0x7 352; VIGFX9_10-NEXT: s_buffer_load_dwordx4 s[{{[0-9]}}:{{[0-9]}}], s[0:3], 0x4 353; VIGFX9_10-NEXT: s_buffer_load_dwordx2 s[{{[0-9]}}:{{[0-9]}}], s[0:3], 0x1c 354define amdgpu_ps void @smrd_imm_merged(<4 x i32> inreg %desc) #0 { 355main_body: 356 %r1 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 4, i32 0) 357 %r2 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 8, i32 0) 358 %r3 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 12, i32 0) 359 %r4 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 16, i32 0) 360 %r5 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 28, i32 0) 361 %r6 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 32, i32 0) 362 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true) #0 363 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float undef, float undef, i1 true, i1 true) #0 364 ret void 365} 366 367; GCN-LABEL: {{^}}smrd_imm_merge_m0: 368; 369; GCN: s_buffer_load_dwordx2 370; SICIVI: s_mov_b32 m0 371; SICIVI_DAG: v_interp_p1_f32 372; SICIVI_DAG: v_interp_p1_f32 373; SICIVI_DAG: v_interp_p1_f32 374; SICIVI_DAG: v_interp_p2_f32 375; SICIVI_DAG: v_interp_p2_f32 376; SICIVI_DAG: v_interp_p2_f32 377; 378; extractelement does not result in movrels anymore for vectors gitting 8 dwords 379; SICIVI-NOT: s_mov_b32 m0 380; SICIVI-NOT: v_movrels_b32_e32 381; v_cndmask_b32_e32 382; v_cndmask_b32_e32 383; 384; Merging is still thwarted on GFX9 due to s_set_gpr_idx 385; 386define amdgpu_ps float @smrd_imm_merge_m0(<4 x i32> inreg %desc, i32 inreg %prim, float %u, float %v) #0 { 387main_body: 388 %idx1.f = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 0, i32 0) 389 %idx1 = bitcast float %idx1.f to i32 390 391 %v0.x1 = call nsz float @llvm.amdgcn.interp.p1(float %u, i32 0, i32 0, i32 %prim) 392 %v0.x = call nsz float @llvm.amdgcn.interp.p2(float %v0.x1, float %v, i32 0, i32 0, i32 %prim) 393 %v0.y1 = call nsz float @llvm.amdgcn.interp.p1(float %u, i32 0, i32 1, i32 %prim) 394 %v0.y = call nsz float @llvm.amdgcn.interp.p2(float %v0.y1, float %v, i32 0, i32 1, i32 %prim) 395 %v0.z1 = call nsz float @llvm.amdgcn.interp.p1(float %u, i32 0, i32 2, i32 %prim) 396 %v0.z = call nsz float @llvm.amdgcn.interp.p2(float %v0.z1, float %v, i32 0, i32 2, i32 %prim) 397 %v0.tmp0 = insertelement <3 x float> undef, float %v0.x, i32 0 398 %v0.tmp1 = insertelement <3 x float> %v0.tmp0, float %v0.y, i32 1 399 %v0 = insertelement <3 x float> %v0.tmp1, float %v0.z, i32 2 400 %a = extractelement <3 x float> %v0, i32 %idx1 401 402 %v1.x1 = call nsz float @llvm.amdgcn.interp.p1(float %u, i32 1, i32 0, i32 %prim) 403 %v1.x = call nsz float @llvm.amdgcn.interp.p2(float %v1.x1, float %v, i32 1, i32 0, i32 %prim) 404 %v1.y1 = call nsz float @llvm.amdgcn.interp.p1(float %u, i32 1, i32 1, i32 %prim) 405 %v1.y = call nsz float @llvm.amdgcn.interp.p2(float %v1.y1, float %v, i32 1, i32 1, i32 %prim) 406 %v1.z1 = call nsz float @llvm.amdgcn.interp.p1(float %u, i32 1, i32 2, i32 %prim) 407 %v1.z = call nsz float @llvm.amdgcn.interp.p2(float %v1.z1, float %v, i32 1, i32 2, i32 %prim) 408 %v1.tmp0 = insertelement <3 x float> undef, float %v0.x, i32 0 409 %v1.tmp1 = insertelement <3 x float> %v0.tmp0, float %v0.y, i32 1 410 %v1 = insertelement <3 x float> %v0.tmp1, float %v0.z, i32 2 411 412 %b = extractelement <3 x float> %v1, i32 %idx1 413 %c = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 4, i32 0) 414 415 %res.tmp = fadd float %a, %b 416 %res = fadd float %res.tmp, %c 417 ret float %res 418} 419 420; GCN-LABEL: {{^}}smrd_vgpr_merged: 421; GCN-NEXT: %bb. 422; GCN-NEXT: buffer_load_dwordx4 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:4 423; GCN-NEXT: buffer_load_dwordx2 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:28 424define amdgpu_ps void @smrd_vgpr_merged(<4 x i32> inreg %desc, i32 %a) #0 { 425main_body: 426 %a1 = add i32 %a, 4 427 %a2 = add i32 %a, 8 428 %a3 = add i32 %a, 12 429 %a4 = add i32 %a, 16 430 %a5 = add i32 %a, 28 431 %a6 = add i32 %a, 32 432 %r1 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %a1, i32 0) 433 %r2 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %a2, i32 0) 434 %r3 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %a3, i32 0) 435 %r4 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %a4, i32 0) 436 %r5 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %a5, i32 0) 437 %r6 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %a6, i32 0) 438 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true) #0 439 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float undef, float undef, i1 true, i1 true) #0 440 ret void 441} 442 443; GCN-LABEL: {{^}}smrd_sgpr_descriptor_promoted 444; GCN: v_readfirstlane 445define amdgpu_cs void @smrd_sgpr_descriptor_promoted([0 x i8] addrspace(4)* inreg noalias dereferenceable(18446744073709551615), i32) #0 { 446main_body: 447 %descptr = bitcast [0 x i8] addrspace(4)* %0 to <4 x i32> addrspace(4)*, !amdgpu.uniform !0 448 br label %.outer_loop_header 449 450ret_block: ; preds = %.outer, %.label22, %main_body 451 ret void 452 453.outer_loop_header: 454 br label %.inner_loop_header 455 456.inner_loop_header: ; preds = %.inner_loop_body, %.outer_loop_header 457 %loopctr.1 = phi i32 [ 0, %.outer_loop_header ], [ %loopctr.2, %.inner_loop_body ] 458 %loopctr.2 = add i32 %loopctr.1, 1 459 %inner_br1 = icmp slt i32 %loopctr.2, 10 460 br i1 %inner_br1, label %.inner_loop_body, label %ret_block 461 462.inner_loop_body: 463 %descriptor = load <4 x i32>, <4 x i32> addrspace(4)* %descptr, align 16, !invariant.load !0 464 %load1result = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %descriptor, i32 0, i32 0) 465 store float %load1result, float addrspace(1)* undef 466 %inner_br2 = icmp uge i32 %1, 10 467 br i1 %inner_br2, label %.inner_loop_header, label %.outer_loop_body 468 469.outer_loop_body: 470 %offset = shl i32 %loopctr.2, 6 471 %load2result = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %descriptor, i32 %offset, i32 0) 472 %outer_br = fcmp ueq float %load2result, 0x0 473 br i1 %outer_br, label %.outer_loop_header, label %ret_block 474} 475 476; SMRD load with a non-const offset 477; GCN-LABEL: {{^}}smrd_load_nonconst0: 478; SIVIGFX9_10: s_buffer_load_dword s{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} 479; SIVIGFX9_10: s_buffer_load_dword s{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} 480; CI: s_buffer_load_dword s{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} 481; CI: s_buffer_load_dword s{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} 482; GCN: s_endpgm 483define amdgpu_ps void @smrd_load_nonconst0(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in, i32 inreg %ncoff) #0 { 484main_body: 485 %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg, i32 0 486 %tmp20 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp 487 %tmp21 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 %ncoff, i32 0) 488 %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in 489 %s.buffer = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %tmp22, i32 %ncoff, i32 0) 490 %s.buffer.float = bitcast i32 %s.buffer to float 491 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %tmp21, float %tmp21, float %tmp21, float %s.buffer.float, i1 true, i1 true) #0 492 ret void 493} 494 495; SMRD load with a non-const non-uniform offset 496; GCN-LABEL: {{^}}smrd_load_nonconst1: 497; SIVIGFX9_10: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen 498; SIVIGFX9_10: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen 499; CI: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen 500; CI: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen 501; GCN: s_endpgm 502define amdgpu_ps void @smrd_load_nonconst1(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in, i32 %ncoff) #0 { 503main_body: 504 %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg, i32 0 505 %tmp20 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp 506 %tmp21 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 %ncoff, i32 0) 507 %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in 508 %s.buffer = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %tmp22, i32 %ncoff, i32 0) 509 %s.buffer.float = bitcast i32 %s.buffer to float 510 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %tmp21, float %tmp21, float %tmp21, float %s.buffer.float, i1 true, i1 true) #0 511 ret void 512} 513 514; SMRD load with a non-const non-uniform offset of > 4 dwords (requires splitting) 515; GCN-LABEL: {{^}}smrd_load_nonconst2: 516; SIVIGFX9_10-DAG: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen 517; SIVIGFX9_10-DAG: buffer_load_dwordx4 v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen 518; CI: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen 519; CI: buffer_load_dwordx4 v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0 offen 520; GCN: s_endpgm 521define amdgpu_ps void @smrd_load_nonconst2(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in, i32 %ncoff) #0 { 522main_body: 523 %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg, i32 0 524 %tmp20 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp 525 %tmp21 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 %ncoff, i32 0) 526 %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in 527 %s.buffer = call <8 x i32> @llvm.amdgcn.s.buffer.load.v8i32(<4 x i32> %tmp22, i32 %ncoff, i32 0) 528 %s.buffer.elt = extractelement <8 x i32> %s.buffer, i32 1 529 %s.buffer.float = bitcast i32 %s.buffer.elt to float 530 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %tmp21, float %tmp21, float %tmp21, float %s.buffer.float, i1 true, i1 true) #0 531 ret void 532} 533 534; SMRD load with a non-const non-uniform offset of > 4 dwords (requires splitting) 535; GCN-LABEL: {{^}}smrd_load_nonconst3: 536; GCN-DAG: buffer_load_dwordx4 v[0:3], v{{[0-9]+}}, s[0:3], 0 offen ; 537; GCN-DAG: buffer_load_dwordx4 v[4:7], v{{[0-9]+}}, s[0:3], 0 offen offset:16 ; 538; GCN-DAG: buffer_load_dwordx4 v[8:11], v{{[0-9]+}}, s[0:3], 0 offen offset:32 ; 539; GCN-DAG: buffer_load_dwordx4 v[12:15], v{{[0-9]+}}, s[0:3], 0 offen offset:48 ; 540; GCN: ; return to shader part epilog 541define amdgpu_ps <16 x float> @smrd_load_nonconst3(<4 x i32> inreg %rsrc, i32 %off) #0 { 542main_body: 543 %ld = call <16 x i32> @llvm.amdgcn.s.buffer.load.v16i32(<4 x i32> %rsrc, i32 %off, i32 0) 544 %bc = bitcast <16 x i32> %ld to <16 x float> 545 ret <16 x float> %bc 546} 547 548; GCN-LABEL: {{^}}smrd_load_nonconst4: 549; SICI: v_add_i32_e32 v{{[0-9]+}}, vcc, 0xff8, v0 ; 550; SICI-DAG: buffer_load_dwordx4 v[0:3], v{{[0-9]+}}, s[0:3], 0 offen ; 551; SICI-DAG: buffer_load_dwordx4 v[4:7], v{{[0-9]+}}, s[0:3], 0 offen offset:16 ; 552; SICI-DAG: buffer_load_dwordx4 v[8:11], v{{[0-9]+}}, s[0:3], 0 offen offset:32 ; 553; SICI-DAG: buffer_load_dwordx4 v[12:15], v{{[0-9]+}}, s[0:3], 0 offen offset:48 ; 554; VIGFX9_10-DAG: buffer_load_dwordx4 v[0:3], v{{[0-9]+}}, s[0:3], 56 offen offset:4032 ; 555; VIGFX9_10-DAG: buffer_load_dwordx4 v[4:7], v{{[0-9]+}}, s[0:3], 56 offen offset:4048 ; 556; VIGFX9_10-DAG: buffer_load_dwordx4 v[8:11], v{{[0-9]+}}, s[0:3], 56 offen offset:4064 ; 557; VIGFX9_10-DAG: buffer_load_dwordx4 v[12:15], v{{[0-9]+}}, s[0:3], 56 offen offset:4080 ; 558; GCN: ; return to shader part epilog 559define amdgpu_ps <16 x float> @smrd_load_nonconst4(<4 x i32> inreg %rsrc, i32 %off) #0 { 560main_body: 561 %off.2 = add i32 %off, 4088 562 %ld = call <16 x i32> @llvm.amdgcn.s.buffer.load.v16i32(<4 x i32> %rsrc, i32 %off.2, i32 0) 563 %bc = bitcast <16 x i32> %ld to <16 x float> 564 ret <16 x float> %bc 565} 566 567; GCN-LABEL: {{^}}smrd_load_nonconst5: 568; SICI: v_add_i32_e32 v{{[0-9]+}}, vcc, 0x1004, v0 569; SICI-DAG: buffer_load_dwordx4 v[0:3], v{{[0-9]+}}, s[0:3], 0 offen ; 570; SICI-DAG: buffer_load_dwordx4 v[4:7], v{{[0-9]+}}, s[0:3], 0 offen offset:16 ; 571; SICI-DAG: buffer_load_dwordx4 v[8:11], v{{[0-9]+}}, s[0:3], 0 offen offset:32 ; 572; SICI-DAG: buffer_load_dwordx4 v[12:15], v{{[0-9]+}}, s[0:3], 0 offen offset:48 ; 573; VIGFX9_10: s_movk_i32 s4, 0xfc0 574; VIGFX9_10-DAG: buffer_load_dwordx4 v[0:3], v{{[0-9]+}}, s[0:3], s4 offen offset:68 ; 575; VIGFX9_10-DAG: buffer_load_dwordx4 v[4:7], v{{[0-9]+}}, s[0:3], s4 offen offset:84 ; 576; VIGFX9_10-DAG: buffer_load_dwordx4 v[8:11], v{{[0-9]+}}, s[0:3], s4 offen offset:100 ; 577; VIGFX9_10-DAG: buffer_load_dwordx4 v[12:15], v{{[0-9]+}}, s[0:3], s4 offen offset:116 ; 578; GCN: ; return to shader part epilog 579define amdgpu_ps <16 x float> @smrd_load_nonconst5(<4 x i32> inreg %rsrc, i32 %off) #0 { 580main_body: 581 %off.2 = add i32 %off, 4100 582 %ld = call <16 x i32> @llvm.amdgcn.s.buffer.load.v16i32(<4 x i32> %rsrc, i32 %off.2, i32 0) 583 %bc = bitcast <16 x i32> %ld to <16 x float> 584 ret <16 x float> %bc 585} 586 587; SMRD load dwordx2 588; GCN-LABEL: {{^}}smrd_load_dwordx2: 589; SIVIGFX9_10: s_buffer_load_dwordx2 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} 590; CI: s_buffer_load_dwordx2 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} 591; GCN: s_endpgm 592define amdgpu_ps void @smrd_load_dwordx2(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, <4 x i32> addrspace(4)* inreg %in, i32 inreg %ncoff) #0 { 593main_body: 594 %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in 595 %s.buffer = call <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32> %tmp22, i32 %ncoff, i32 0) 596 %s.buffer.float = bitcast <2 x i32> %s.buffer to <2 x float> 597 %r.1 = extractelement <2 x float> %s.buffer.float, i32 0 598 %r.2 = extractelement <2 x float> %s.buffer.float, i32 1 599 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r.1, float %r.1, float %r.1, float %r.2, i1 true, i1 true) #0 600 ret void 601} 602 603; GCN-LABEL: {{^}}smrd_uniform_loop: 604; 605; TODO: we should keep the loop counter in an SGPR 606; 607; GCN: s_buffer_load_dword 608define amdgpu_ps float @smrd_uniform_loop(<4 x i32> inreg %desc, i32 %bound) #0 { 609main_body: 610 br label %loop 611 612loop: 613 %counter = phi i32 [ 0, %main_body ], [ %counter.next, %loop ] 614 %sum = phi float [ 0.0, %main_body ], [ %sum.next, %loop ] 615 %offset = shl i32 %counter, 2 616 %v = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %offset, i32 0) 617 %sum.next = fadd float %sum, %v 618 %counter.next = add i32 %counter, 1 619 %cc = icmp uge i32 %counter.next, %bound 620 br i1 %cc, label %exit, label %loop 621 622exit: 623 ret float %sum.next 624} 625 626 627; GCN-LABEL: {{^}}smrd_uniform_loop2: 628; (this test differs from smrd_uniform_loop by the more complex structure of phis, 629; which used to confuse the DivergenceAnalysis after structurization) 630; 631; TODO: we should keep the loop counter in an SGPR and use an S_BUFFER_LOAD 632; 633; GCN: buffer_load_dword 634define amdgpu_ps float @smrd_uniform_loop2(<4 x i32> inreg %desc, i32 %bound, i32 %bound.a) #0 { 635main_body: 636 br label %loop 637 638loop: 639 %counter = phi i32 [ 0, %main_body ], [ %counter.next, %loop.a ], [ %counter.next, %loop.b ] 640 %sum = phi float [ 0.0, %main_body ], [ %sum.next, %loop.a ], [ %sum.next.b, %loop.b ] 641 %offset = shl i32 %counter, 2 642 %v = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %offset, i32 0) 643 %sum.next = fadd float %sum, %v 644 %counter.next = add i32 %counter, 1 645 %cc = icmp uge i32 %counter.next, %bound 646 br i1 %cc, label %exit, label %loop.a 647 648loop.a: 649 %cc.a = icmp uge i32 %counter.next, %bound.a 650 br i1 %cc, label %loop, label %loop.b 651 652loop.b: 653 %sum.next.b = fadd float %sum.next, 1.0 654 br label %loop 655 656exit: 657 ret float %sum.next 658} 659 660; This test checks that the load after some control flow with an offset based 661; on a divergent shader input is correctly recognized as divergent. This was 662; reduced from an actual regression. Yes, the %unused argument matters, as 663; well as the fact that %arg4 is a vector. 664; 665; GCN-LABEL: {{^}}arg_divergence: 666; GCN: buffer_load_dword v0, v0, 667; GCN-NEXT: s_waitcnt 668; GCN-NEXT: ; return to shader part epilog 669define amdgpu_cs float @arg_divergence(i32 inreg %unused, <3 x i32> %arg4) #0 { 670main_body: 671 br i1 undef, label %if1, label %endif1 672 673if1: ; preds = %main_body 674 store i32 0, i32 addrspace(3)* undef, align 4 675 br label %endif1 676 677endif1: ; preds = %if1, %main_body 678 %tmp13 = extractelement <3 x i32> %arg4, i32 0 679 %tmp97 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 %tmp13, i32 0) 680 ret float %tmp97 681} 682 683; GCN-LABEL: {{^}}s_buffer_load_f32: 684; GCN: s_buffer_load_dword s0, s[0:3], s4 685define amdgpu_ps void @s_buffer_load_f32(<4 x i32> inreg %rsrc, i32 inreg %offset) { 686 %sgpr = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %rsrc, i32 %offset, i32 0) 687 call void asm sideeffect "; use $0", "s"(float %sgpr) 688 ret void 689} 690 691; GCN-LABEL: {{^}}s_buffer_load_v2f32: 692; GCN: s_buffer_load_dwordx2 s[0:1], s[0:3], s4 693define amdgpu_ps void @s_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 inreg %offset) { 694 %sgpr = call <2 x float> @llvm.amdgcn.s.buffer.load.v2f32(<4 x i32> %rsrc, i32 %offset, i32 0) 695 call void asm sideeffect "; use $0", "s"(<2 x float> %sgpr) 696 ret void 697} 698 699; GCN-LABEL: {{^}}s_buffer_load_v4f32: 700; GCN: s_buffer_load_dwordx4 s[0:3], s[0:3], s4 701define amdgpu_ps void @s_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 inreg %offset) { 702 %sgpr = call <4 x float> @llvm.amdgcn.s.buffer.load.v4f32(<4 x i32> %rsrc, i32 %offset, i32 0) 703 call void asm sideeffect "; use $0", "s"(<4 x float> %sgpr) 704 ret void 705} 706 707; GCN-LABEL: {{^}}s_buffer_load_v8f32: 708; GCN: s_buffer_load_dwordx8 s[0:7], s[0:3], s4 709define amdgpu_ps void @s_buffer_load_v8f32(<4 x i32> inreg %rsrc, i32 inreg %offset) { 710 %sgpr = call <8 x float> @llvm.amdgcn.s.buffer.load.v8f32(<4 x i32> %rsrc, i32 %offset, i32 0) 711 call void asm sideeffect "; use $0", "s"(<8 x float> %sgpr) 712 ret void 713} 714 715; GCN-LABEL: {{^}}s_buffer_load_v16f32: 716; GCN: s_buffer_load_dwordx16 s[0:15], s[0:3], s4 717define amdgpu_ps void @s_buffer_load_v16f32(<4 x i32> inreg %rsrc, i32 inreg %offset) { 718 %sgpr = call <16 x float> @llvm.amdgcn.s.buffer.load.v16f32(<4 x i32> %rsrc, i32 %offset, i32 0) 719 call void asm sideeffect "; use $0", "s"(<16 x float> %sgpr) 720 ret void 721} 722 723declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #0 724declare float @llvm.amdgcn.interp.p1(float, i32, i32, i32) #2 725declare float @llvm.amdgcn.interp.p2(float, float, i32, i32, i32) #2 726 727declare i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32>, i32, i32) #1 728declare <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32>, i32, i32) 729declare <4 x i32> @llvm.amdgcn.s.buffer.load.v4i32(<4 x i32>, i32, i32) 730declare <8 x i32> @llvm.amdgcn.s.buffer.load.v8i32(<4 x i32>, i32, i32) 731declare <16 x i32> @llvm.amdgcn.s.buffer.load.v16i32(<4 x i32>, i32, i32) 732 733declare float @llvm.amdgcn.s.buffer.load.f32(<4 x i32>, i32, i32) 734declare <2 x float> @llvm.amdgcn.s.buffer.load.v2f32(<4 x i32>, i32, i32) 735declare <4 x float> @llvm.amdgcn.s.buffer.load.v4f32(<4 x i32>, i32, i32) 736declare <8 x float> @llvm.amdgcn.s.buffer.load.v8f32(<4 x i32>, i32, i32) 737declare <16 x float> @llvm.amdgcn.s.buffer.load.v16f32(<4 x i32>, i32, i32) 738 739attributes #0 = { nounwind } 740attributes #1 = { nounwind readnone } 741attributes #2 = { nounwind readnone speculatable } 742 743!0 = !{} 744