1; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,CI %s 2; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=+load-store-opt,+flat-for-global < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,GFX9 %s 3 4; FIXME: We don't get cases where the address was an SGPR because we 5; get a copy to the address register for each one. 6 7@lds = addrspace(3) global [512 x float] undef, align 4 8@lds.f64 = addrspace(3) global [512 x double] undef, align 8 9 10; GCN-LABEL: {{^}}simple_read2_f32: 11; CI-DAG: s_mov_b32 m0 12; GFX9-NOT: m0 13 14; GCN: ds_read2_b32 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:8 15; GCN: s_waitcnt lgkmcnt(0) 16; GCN: v_add_f32_e32 [[RESULT:v[0-9]+]], v[[LO_VREG]], v[[HI_VREG]] 17; CI: buffer_store_dword [[RESULT]] 18; GFX9: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] 19; GCN: s_endpgm 20define amdgpu_kernel void @simple_read2_f32(float addrspace(1)* %out) #0 { 21 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 22 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i 23 %val0 = load float, float addrspace(3)* %arrayidx0, align 4 24 %add.x = add nsw i32 %x.i, 8 25 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %add.x 26 %val1 = load float, float addrspace(3)* %arrayidx1, align 4 27 %sum = fadd float %val0, %val1 28 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 29 store float %sum, float addrspace(1)* %out.gep, align 4 30 ret void 31} 32 33; GCN-LABEL: {{^}}simple_read2_f32_max_offset: 34; CI-DAG: s_mov_b32 m0 35; GFX9-NOT: m0 36 37; GCN: ds_read2_b32 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:255 38; GCN: s_waitcnt lgkmcnt(0) 39; GCN: v_add_f32_e32 [[RESULT:v[0-9]+]], v[[LO_VREG]], v[[HI_VREG]] 40 41; CI: buffer_store_dword [[RESULT]] 42; GFX9: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] 43define amdgpu_kernel void @simple_read2_f32_max_offset(float addrspace(1)* %out) #0 { 44 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 45 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i 46 %val0 = load float, float addrspace(3)* %arrayidx0, align 4 47 %add.x = add nsw i32 %x.i, 255 48 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %add.x 49 %val1 = load float, float addrspace(3)* %arrayidx1, align 4 50 %sum = fadd float %val0, %val1 51 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 52 store float %sum, float addrspace(1)* %out.gep, align 4 53 ret void 54} 55 56; GCN-LABEL: @simple_read2_f32_too_far 57; CI-DAG: s_mov_b32 m0 58; GFX9-NOT: m0 59 60; GCN-NOT: ds_read2_b32 61; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} 62; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:1028 63; GCN: s_endpgm 64define amdgpu_kernel void @simple_read2_f32_too_far(float addrspace(1)* %out) #0 { 65 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 66 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i 67 %val0 = load float, float addrspace(3)* %arrayidx0, align 4 68 %add.x = add nsw i32 %x.i, 257 69 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %add.x 70 %val1 = load float, float addrspace(3)* %arrayidx1, align 4 71 %sum = fadd float %val0, %val1 72 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 73 store float %sum, float addrspace(1)* %out.gep, align 4 74 ret void 75} 76 77; GCN-LABEL: @simple_read2_f32_x2 78; CI-DAG: s_mov_b32 m0 79; GFX9-NOT: m0 80 81; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR:v[0-9]+]] offset1:8 82; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR]] offset0:11 offset1:27 83; GCN: s_endpgm 84define amdgpu_kernel void @simple_read2_f32_x2(float addrspace(1)* %out) #0 { 85 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 86 %idx.0 = add nsw i32 %tid.x, 0 87 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.0 88 %val0 = load float, float addrspace(3)* %arrayidx0, align 4 89 90 %idx.1 = add nsw i32 %tid.x, 8 91 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.1 92 %val1 = load float, float addrspace(3)* %arrayidx1, align 4 93 %sum.0 = fadd float %val0, %val1 94 95 %idx.2 = add nsw i32 %tid.x, 11 96 %arrayidx2 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.2 97 %val2 = load float, float addrspace(3)* %arrayidx2, align 4 98 99 %idx.3 = add nsw i32 %tid.x, 27 100 %arrayidx3 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.3 101 %val3 = load float, float addrspace(3)* %arrayidx3, align 4 102 %sum.1 = fadd float %val2, %val3 103 104 %sum = fadd float %sum.0, %sum.1 105 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %idx.0 106 store float %sum, float addrspace(1)* %out.gep, align 4 107 ret void 108} 109 110; Make sure there is an instruction between the two sets of reads. 111; GCN-LABEL: @simple_read2_f32_x2_barrier 112; CI-DAG: s_mov_b32 m0 113; GFX9-NOT: m0 114 115; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR:v[0-9]+]] offset1:8 116; GCN: s_barrier 117; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR]] offset0:11 offset1:27 118; GCN: s_endpgm 119define amdgpu_kernel void @simple_read2_f32_x2_barrier(float addrspace(1)* %out) #0 { 120 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 121 %idx.0 = add nsw i32 %tid.x, 0 122 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.0 123 %val0 = load float, float addrspace(3)* %arrayidx0, align 4 124 125 %idx.1 = add nsw i32 %tid.x, 8 126 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.1 127 %val1 = load float, float addrspace(3)* %arrayidx1, align 4 128 %sum.0 = fadd float %val0, %val1 129 130 call void @llvm.amdgcn.s.barrier() #2 131 132 %idx.2 = add nsw i32 %tid.x, 11 133 %arrayidx2 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.2 134 %val2 = load float, float addrspace(3)* %arrayidx2, align 4 135 136 %idx.3 = add nsw i32 %tid.x, 27 137 %arrayidx3 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.3 138 %val3 = load float, float addrspace(3)* %arrayidx3, align 4 139 %sum.1 = fadd float %val2, %val3 140 141 %sum = fadd float %sum.0, %sum.1 142 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %idx.0 143 store float %sum, float addrspace(1)* %out.gep, align 4 144 ret void 145} 146 147; For some reason adding something to the base address for the first 148; element results in only folding the inner pair. 149 150; GCN-LABEL: @simple_read2_f32_x2_nonzero_base 151; CI-DAG: s_mov_b32 m0 152; GFX9-NOT: m0 153 154; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR:v[0-9]+]] offset0:2 offset1:8 155; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR]] offset0:11 offset1:27 156; GCN: s_endpgm 157define amdgpu_kernel void @simple_read2_f32_x2_nonzero_base(float addrspace(1)* %out) #0 { 158 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 159 %idx.0 = add nsw i32 %tid.x, 2 160 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.0 161 %val0 = load float, float addrspace(3)* %arrayidx0, align 4 162 163 %idx.1 = add nsw i32 %tid.x, 8 164 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.1 165 %val1 = load float, float addrspace(3)* %arrayidx1, align 4 166 %sum.0 = fadd float %val0, %val1 167 168 %idx.2 = add nsw i32 %tid.x, 11 169 %arrayidx2 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.2 170 %val2 = load float, float addrspace(3)* %arrayidx2, align 4 171 172 %idx.3 = add nsw i32 %tid.x, 27 173 %arrayidx3 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.3 174 %val3 = load float, float addrspace(3)* %arrayidx3, align 4 175 %sum.1 = fadd float %val2, %val3 176 177 %sum = fadd float %sum.0, %sum.1 178 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %idx.0 179 store float %sum, float addrspace(1)* %out.gep, align 4 180 ret void 181} 182 183; Be careful of vectors of pointers. We don't know if the 2 pointers 184; in the vectors are really the same base, so this is not safe to 185; merge. 186; Base pointers come from different subregister of same super 187; register. We can't safely merge this. 188 189; GCN-LABEL: @read2_ptr_is_subreg_arg_f32 190; CI-DAG: s_mov_b32 m0 191; GFX9-NOT: m0 192 193; GCN-NOT: ds_read2_b32 194; GCN: ds_read_b32 195; GCN: ds_read_b32 196; GCN: s_endpgm 197define amdgpu_kernel void @read2_ptr_is_subreg_arg_f32(float addrspace(1)* %out, <2 x float addrspace(3)*> %lds.ptr) #0 { 198 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 199 %index.0 = insertelement <2 x i32> undef, i32 %x.i, i32 0 200 %index.1 = insertelement <2 x i32> %index.0, i32 8, i32 0 201 %gep = getelementptr inbounds float, <2 x float addrspace(3)*> %lds.ptr, <2 x i32> %index.1 202 %gep.0 = extractelement <2 x float addrspace(3)*> %gep, i32 0 203 %gep.1 = extractelement <2 x float addrspace(3)*> %gep, i32 1 204 %val0 = load float, float addrspace(3)* %gep.0, align 4 205 %val1 = load float, float addrspace(3)* %gep.1, align 4 206 %add.x = add nsw i32 %x.i, 8 207 %sum = fadd float %val0, %val1 208 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 209 store float %sum, float addrspace(1)* %out.gep, align 4 210 ret void 211} 212 213; Apply a constant scalar offset after the pointer vector extract. We 214; are rejecting merges that have the same, constant 0 offset, so make 215; sure we are really rejecting it because of the different 216; subregisters. 217 218; GCN-LABEL: @read2_ptr_is_subreg_arg_offset_f32 219; CI-DAG: s_mov_b32 m0 220; GFX9-NOT: m0 221 222; GCN-NOT: ds_read2_b32 223; GCN: ds_read_b32 224; GCN: ds_read_b32 225; GCN: s_endpgm 226define amdgpu_kernel void @read2_ptr_is_subreg_arg_offset_f32(float addrspace(1)* %out, <2 x float addrspace(3)*> %lds.ptr) #0 { 227 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 228 %index.0 = insertelement <2 x i32> undef, i32 %x.i, i32 0 229 %index.1 = insertelement <2 x i32> %index.0, i32 8, i32 0 230 %gep = getelementptr inbounds float, <2 x float addrspace(3)*> %lds.ptr, <2 x i32> %index.1 231 %gep.0 = extractelement <2 x float addrspace(3)*> %gep, i32 0 232 %gep.1 = extractelement <2 x float addrspace(3)*> %gep, i32 1 233 234 ; Apply an additional offset after the vector that will be more obviously folded. 235 %gep.1.offset = getelementptr float, float addrspace(3)* %gep.1, i32 8 236 237 %val0 = load float, float addrspace(3)* %gep.0, align 4 238 %val1 = load float, float addrspace(3)* %gep.1.offset, align 4 239 %add.x = add nsw i32 %x.i, 8 240 %sum = fadd float %val0, %val1 241 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 242 store float %sum, float addrspace(1)* %out.gep, align 4 243 ret void 244} 245 246; GCN-LABEL: {{^}}read2_ptr_is_subreg_f32: 247; CI-DAG: s_mov_b32 m0 248; GFX9-NOT: m0 249 250; GCN: ds_read2_b32 {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset1:8{{$}} 251; GCN: s_endpgm 252define amdgpu_kernel void @read2_ptr_is_subreg_f32(float addrspace(1)* %out) #0 { 253 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 254 %ptr.0 = insertelement <2 x [512 x float] addrspace(3)*> undef, [512 x float] addrspace(3)* @lds, i32 0 255 %ptr.1 = insertelement <2 x [512 x float] addrspace(3)*> %ptr.0, [512 x float] addrspace(3)* @lds, i32 1 256 %x.i.v.0 = insertelement <2 x i32> undef, i32 %x.i, i32 0 257 %x.i.v.1 = insertelement <2 x i32> %x.i.v.0, i32 %x.i, i32 1 258 %idx = add <2 x i32> %x.i.v.1, <i32 0, i32 8> 259 %gep = getelementptr inbounds [512 x float], <2 x [512 x float] addrspace(3)*> %ptr.1, <2 x i32> <i32 0, i32 0>, <2 x i32> %idx 260 %gep.0 = extractelement <2 x float addrspace(3)*> %gep, i32 0 261 %gep.1 = extractelement <2 x float addrspace(3)*> %gep, i32 1 262 %val0 = load float, float addrspace(3)* %gep.0, align 4 263 %val1 = load float, float addrspace(3)* %gep.1, align 4 264 %add.x = add nsw i32 %x.i, 8 265 %sum = fadd float %val0, %val1 266 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 267 store float %sum, float addrspace(1)* %out.gep, align 4 268 ret void 269} 270 271; GCN-LABEL: @simple_read2_f32_volatile_0 272; CI-DAG: s_mov_b32 m0 273; GFX9-NOT: m0 274 275; GCN-NOT: ds_read2_b32 276; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} 277; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:32 278; GCN: s_endpgm 279define amdgpu_kernel void @simple_read2_f32_volatile_0(float addrspace(1)* %out) #0 { 280 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 281 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i 282 %val0 = load volatile float, float addrspace(3)* %arrayidx0, align 4 283 %add.x = add nsw i32 %x.i, 8 284 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %add.x 285 %val1 = load float, float addrspace(3)* %arrayidx1, align 4 286 %sum = fadd float %val0, %val1 287 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 288 store float %sum, float addrspace(1)* %out.gep, align 4 289 ret void 290} 291 292; GCN-LABEL: @simple_read2_f32_volatile_1 293; CI-DAG: s_mov_b32 m0 294; GFX9-NOT: m0 295 296; GCN-NOT: ds_read2_b32 297; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} 298; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:32 299; GCN: s_endpgm 300define amdgpu_kernel void @simple_read2_f32_volatile_1(float addrspace(1)* %out) #0 { 301 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 302 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i 303 %val0 = load float, float addrspace(3)* %arrayidx0, align 4 304 %add.x = add nsw i32 %x.i, 8 305 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %add.x 306 %val1 = load volatile float, float addrspace(3)* %arrayidx1, align 4 307 %sum = fadd float %val0, %val1 308 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 309 store float %sum, float addrspace(1)* %out.gep, align 4 310 ret void 311} 312 313; Can't fold since not correctly aligned. 314; XXX: This isn't really testing anything useful now. I think CI 315; allows unaligned LDS accesses, which would be a problem here. 316; GCN-LABEL: @unaligned_read2_f32 317; CI-DAG: s_mov_b32 m0 318; GFX9-NOT: m0 319 320; GCN-NOT: ds_read2_b32 321; GCN: s_endpgm 322define amdgpu_kernel void @unaligned_read2_f32(float addrspace(1)* %out, float addrspace(3)* %lds) #0 { 323 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 324 %arrayidx0 = getelementptr inbounds float, float addrspace(3)* %lds, i32 %x.i 325 %val0 = load float, float addrspace(3)* %arrayidx0, align 1 326 %add.x = add nsw i32 %x.i, 8 327 %arrayidx1 = getelementptr inbounds float, float addrspace(3)* %lds, i32 %add.x 328 %val1 = load float, float addrspace(3)* %arrayidx1, align 1 329 %sum = fadd float %val0, %val1 330 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 331 store float %sum, float addrspace(1)* %out.gep, align 4 332 ret void 333} 334 335; GCN-LABEL: @misaligned_2_simple_read2_f32 336; CI-DAG: s_mov_b32 m0 337; GFX9-NOT: m0 338 339; GCN-NOT: ds_read2_b32 340; GCN: s_endpgm 341define amdgpu_kernel void @misaligned_2_simple_read2_f32(float addrspace(1)* %out, float addrspace(3)* %lds) #0 { 342 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 343 %arrayidx0 = getelementptr inbounds float, float addrspace(3)* %lds, i32 %x.i 344 %val0 = load float, float addrspace(3)* %arrayidx0, align 2 345 %add.x = add nsw i32 %x.i, 8 346 %arrayidx1 = getelementptr inbounds float, float addrspace(3)* %lds, i32 %add.x 347 %val1 = load float, float addrspace(3)* %arrayidx1, align 2 348 %sum = fadd float %val0, %val1 349 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 350 store float %sum, float addrspace(1)* %out.gep, align 4 351 ret void 352} 353 354; GCN-LABEL: @simple_read2_f64 355; CI-DAG: s_mov_b32 m0 356; GFX9-NOT: m0 357 358; GCN-DAG: v_lshlrev_b32_e32 [[VOFS:v[0-9]+]], 3, {{v[0-9]+}} 359; GCN-DAG: v_add_{{[iu]}}32_e32 [[VPTR:v[0-9]+]], {{(vcc, )?}}lds.f64@abs32@lo, [[VOFS]] 360; GCN: ds_read2_b64 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, [[VPTR]] offset1:8 361; GCN: v_add_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], v{{\[}}[[LO_VREG]]:{{[0-9]+\]}}, v{{\[[0-9]+}}:[[HI_VREG]]{{\]}} 362 363; CI: buffer_store_dwordx2 [[RESULT]] 364; GFX9: global_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] 365define amdgpu_kernel void @simple_read2_f64(double addrspace(1)* %out) #0 { 366 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 367 %arrayidx0 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %x.i 368 %val0 = load double, double addrspace(3)* %arrayidx0, align 8 369 %add.x = add nsw i32 %x.i, 8 370 %arrayidx1 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %add.x 371 %val1 = load double, double addrspace(3)* %arrayidx1, align 8 372 %sum = fadd double %val0, %val1 373 %out.gep = getelementptr inbounds double, double addrspace(1)* %out, i32 %x.i 374 store double %sum, double addrspace(1)* %out.gep, align 8 375 ret void 376} 377 378; GCN-LABEL: @simple_read2_f64_max_offset 379; CI-DAG: s_mov_b32 m0 380; GFX9-NOT: m0 381 382; GCN: ds_read2_b64 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:255 383; GCN: s_endpgm 384define amdgpu_kernel void @simple_read2_f64_max_offset(double addrspace(1)* %out) #0 { 385 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 386 %arrayidx0 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %x.i 387 %val0 = load double, double addrspace(3)* %arrayidx0, align 8 388 %add.x = add nsw i32 %x.i, 255 389 %arrayidx1 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %add.x 390 %val1 = load double, double addrspace(3)* %arrayidx1, align 8 391 %sum = fadd double %val0, %val1 392 %out.gep = getelementptr inbounds double, double addrspace(1)* %out, i32 %x.i 393 store double %sum, double addrspace(1)* %out.gep, align 8 394 ret void 395} 396 397; GCN-LABEL: @simple_read2_f64_too_far 398; CI-DAG: s_mov_b32 m0 399; GFX9-NOT: m0 400 401; GCN-NOT: ds_read2_b64 402; GCN: ds_read_b64 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} 403; GCN: ds_read_b64 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:2056 404; GCN: s_endpgm 405define amdgpu_kernel void @simple_read2_f64_too_far(double addrspace(1)* %out) #0 { 406 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 407 %arrayidx0 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %x.i 408 %val0 = load double, double addrspace(3)* %arrayidx0, align 8 409 %add.x = add nsw i32 %x.i, 257 410 %arrayidx1 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %add.x 411 %val1 = load double, double addrspace(3)* %arrayidx1, align 8 412 %sum = fadd double %val0, %val1 413 %out.gep = getelementptr inbounds double, double addrspace(1)* %out, i32 %x.i 414 store double %sum, double addrspace(1)* %out.gep, align 8 415 ret void 416} 417 418; Alignment only 4 419; GCN-LABEL: @misaligned_read2_f64 420; CI-DAG: s_mov_b32 m0 421; GFX9-NOT: m0 422 423; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset1:1 424; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset0:14 offset1:15 425; GCN: s_endpgm 426define amdgpu_kernel void @misaligned_read2_f64(double addrspace(1)* %out, double addrspace(3)* %lds) #0 { 427 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 428 %arrayidx0 = getelementptr inbounds double, double addrspace(3)* %lds, i32 %x.i 429 %val0 = load double, double addrspace(3)* %arrayidx0, align 4 430 %add.x = add nsw i32 %x.i, 7 431 %arrayidx1 = getelementptr inbounds double, double addrspace(3)* %lds, i32 %add.x 432 %val1 = load double, double addrspace(3)* %arrayidx1, align 4 433 %sum = fadd double %val0, %val1 434 %out.gep = getelementptr inbounds double, double addrspace(1)* %out, i32 %x.i 435 store double %sum, double addrspace(1)* %out.gep, align 4 436 ret void 437} 438 439@foo = addrspace(3) global [4 x i32] undef, align 4 440 441; GCN-LABEL: @load_constant_adjacent_offsets 442; CI-DAG: s_mov_b32 m0 443; GFX9-NOT: m0 444 445; GCN-DAG: v_mov_b32_e32 [[PTR:v[0-9]+]], foo@abs32@lo{{$}} 446; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]] offset1:1 447define amdgpu_kernel void @load_constant_adjacent_offsets(i32 addrspace(1)* %out) { 448 %val0 = load i32, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 0), align 4 449 %val1 = load i32, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 1), align 4 450 %sum = add i32 %val0, %val1 451 store i32 %sum, i32 addrspace(1)* %out, align 4 452 ret void 453} 454 455; GCN-LABEL: @load_constant_disjoint_offsets 456; CI-DAG: s_mov_b32 m0 457; GFX9-NOT: m0 458 459; GCN-DAG: v_mov_b32_e32 [[PTR:v[0-9]+]], foo@abs32@lo{{$}} 460; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]] offset1:2 461define amdgpu_kernel void @load_constant_disjoint_offsets(i32 addrspace(1)* %out) { 462 %val0 = load i32, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 0), align 4 463 %val1 = load i32, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 2), align 4 464 %sum = add i32 %val0, %val1 465 store i32 %sum, i32 addrspace(1)* %out, align 4 466 ret void 467} 468 469@bar = addrspace(3) global [4 x i64] undef, align 4 470 471; GCN-LABEL: @load_misaligned64_constant_offsets 472; CI-DAG: s_mov_b32 m0 473; GFX9-NOT: m0 474 475; GCN-DAG: v_mov_b32_e32 [[PTR:v[0-9]+]], bar@abs32@lo{{$}} 476; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]] offset0:2 offset1:3 477; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]] offset1:1 478define amdgpu_kernel void @load_misaligned64_constant_offsets(i64 addrspace(1)* %out) { 479 %val0 = load i64, i64 addrspace(3)* getelementptr inbounds ([4 x i64], [4 x i64] addrspace(3)* @bar, i32 0, i32 0), align 4 480 %val1 = load i64, i64 addrspace(3)* getelementptr inbounds ([4 x i64], [4 x i64] addrspace(3)* @bar, i32 0, i32 1), align 4 481 %sum = add i64 %val0, %val1 482 store i64 %sum, i64 addrspace(1)* %out, align 8 483 ret void 484} 485 486@bar.large = addrspace(3) global [4096 x i64] undef, align 4 487 488; GCN-LABEL: @load_misaligned64_constant_large_offsets 489; CI-DAG: s_mov_b32 m0 490; GFX9-NOT: m0 491 492; GCN-DAG: s_mov_b32 [[SBASE0:s[0-9]+]], bar.large@abs32@lo 493; GCN-DAG: s_add_i32 [[SBASE1:s[0-9]+]], [[SBASE0]], 0x4000{{$}} 494; GCN-DAG: s_addk_i32 [[SBASE0]], 0x7ff8{{$}} 495; GCN-DAG: v_mov_b32_e32 [[VBASE0:v[0-9]+]], [[SBASE0]] 496; GCN-DAG: v_mov_b32_e32 [[VBASE1:v[0-9]+]], [[SBASE1]] 497; GCN-DAG: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[VBASE0]] offset1:1 498; GCN-DAG: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[VBASE1]] offset1:1 499; GCN: s_endpgm 500define amdgpu_kernel void @load_misaligned64_constant_large_offsets(i64 addrspace(1)* %out) { 501 %val0 = load i64, i64 addrspace(3)* getelementptr inbounds ([4096 x i64], [4096 x i64] addrspace(3)* @bar.large, i32 0, i32 2048), align 4 502 %val1 = load i64, i64 addrspace(3)* getelementptr inbounds ([4096 x i64], [4096 x i64] addrspace(3)* @bar.large, i32 0, i32 4095), align 4 503 %sum = add i64 %val0, %val1 504 store i64 %sum, i64 addrspace(1)* %out, align 8 505 ret void 506} 507 508@sgemm.lA = internal unnamed_addr addrspace(3) global [264 x float] undef, align 4 509@sgemm.lB = internal unnamed_addr addrspace(3) global [776 x float] undef, align 4 510 511; GCN-LABEL: {{^}}sgemm_inner_loop_read2_sequence: 512; CI-DAG: s_mov_b32 m0 513; GFX9-NOT: m0 514 515define amdgpu_kernel void @sgemm_inner_loop_read2_sequence(float addrspace(1)* %C, i32 %lda, i32 %ldb) #0 { 516 %x.i = tail call i32 @llvm.amdgcn.workgroup.id.x() #1 517 %y.i = tail call i32 @llvm.amdgcn.workitem.id.y() #1 518 %arrayidx44 = getelementptr inbounds [264 x float], [264 x float] addrspace(3)* @sgemm.lA, i32 0, i32 %x.i 519 %tmp16 = load float, float addrspace(3)* %arrayidx44, align 4 520 %add47 = add nsw i32 %x.i, 1 521 %arrayidx48 = getelementptr inbounds [264 x float], [264 x float] addrspace(3)* @sgemm.lA, i32 0, i32 %add47 522 %tmp17 = load float, float addrspace(3)* %arrayidx48, align 4 523 %add51 = add nsw i32 %x.i, 16 524 %arrayidx52 = getelementptr inbounds [264 x float], [264 x float] addrspace(3)* @sgemm.lA, i32 0, i32 %add51 525 %tmp18 = load float, float addrspace(3)* %arrayidx52, align 4 526 %add55 = add nsw i32 %x.i, 17 527 %arrayidx56 = getelementptr inbounds [264 x float], [264 x float] addrspace(3)* @sgemm.lA, i32 0, i32 %add55 528 %tmp19 = load float, float addrspace(3)* %arrayidx56, align 4 529 %arrayidx60 = getelementptr inbounds [776 x float], [776 x float] addrspace(3)* @sgemm.lB, i32 0, i32 %y.i 530 %tmp20 = load float, float addrspace(3)* %arrayidx60, align 4 531 %add63 = add nsw i32 %y.i, 1 532 %arrayidx64 = getelementptr inbounds [776 x float], [776 x float] addrspace(3)* @sgemm.lB, i32 0, i32 %add63 533 %tmp21 = load float, float addrspace(3)* %arrayidx64, align 4 534 %add67 = add nsw i32 %y.i, 32 535 %arrayidx68 = getelementptr inbounds [776 x float], [776 x float] addrspace(3)* @sgemm.lB, i32 0, i32 %add67 536 %tmp22 = load float, float addrspace(3)* %arrayidx68, align 4 537 %add71 = add nsw i32 %y.i, 33 538 %arrayidx72 = getelementptr inbounds [776 x float], [776 x float] addrspace(3)* @sgemm.lB, i32 0, i32 %add71 539 %tmp23 = load float, float addrspace(3)* %arrayidx72, align 4 540 %add75 = add nsw i32 %y.i, 64 541 %arrayidx76 = getelementptr inbounds [776 x float], [776 x float] addrspace(3)* @sgemm.lB, i32 0, i32 %add75 542 %tmp24 = load float, float addrspace(3)* %arrayidx76, align 4 543 %add79 = add nsw i32 %y.i, 65 544 %arrayidx80 = getelementptr inbounds [776 x float], [776 x float] addrspace(3)* @sgemm.lB, i32 0, i32 %add79 545 %tmp25 = load float, float addrspace(3)* %arrayidx80, align 4 546 %sum.0 = fadd float %tmp16, %tmp17 547 %sum.1 = fadd float %sum.0, %tmp18 548 %sum.2 = fadd float %sum.1, %tmp19 549 %sum.3 = fadd float %sum.2, %tmp20 550 %sum.4 = fadd float %sum.3, %tmp21 551 %sum.5 = fadd float %sum.4, %tmp22 552 %sum.6 = fadd float %sum.5, %tmp23 553 %sum.7 = fadd float %sum.6, %tmp24 554 %sum.8 = fadd float %sum.7, %tmp25 555 store float %sum.8, float addrspace(1)* %C, align 4 556 ret void 557} 558 559; GCN-LABEL: {{^}}misaligned_read2_v2i32: 560; CI-DAG: s_mov_b32 m0 561; GFX9-NOT: m0 562define amdgpu_kernel void @misaligned_read2_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(3)* %in) #0 { 563 %load = load <2 x i32>, <2 x i32> addrspace(3)* %in, align 4 564 store <2 x i32> %load, <2 x i32> addrspace(1)* %out, align 8 565 ret void 566} 567 568; GCN-LABEL: {{^}}misaligned_read2_i64: 569; CI-DAG: s_mov_b32 m0 570; GFX9-NOT: m0 571define amdgpu_kernel void @misaligned_read2_i64(i64 addrspace(1)* %out, i64 addrspace(3)* %in) #0 { 572 %load = load i64, i64 addrspace(3)* %in, align 4 573 store i64 %load, i64 addrspace(1)* %out, align 8 574 ret void 575} 576 577; GCN-LABEL: ds_read_diff_base_interleaving 578; CI-DAG: s_mov_b32 m0 579; GFX9-NOT: m0 580 581; GCN-NOT: ds_read_b32 582define amdgpu_kernel void @ds_read_diff_base_interleaving( 583 float addrspace(1)* nocapture %arg, 584 [4 x [4 x float]] addrspace(3)* %arg1, 585 [4 x [4 x float]] addrspace(3)* %arg2, 586 [4 x [4 x float]] addrspace(3)* %arg3, 587 [4 x [4 x float]] addrspace(3)* %arg4) #1 { 588bb: 589 %tmp = getelementptr float, float addrspace(1)* %arg, i64 10 590 %tmp5 = tail call i32 @llvm.amdgcn.workitem.id.x() #2 591 %tmp6 = tail call i32 @llvm.amdgcn.workitem.id.y() #2 592 %tmp7 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg1, i32 0, i32 %tmp6, i32 0 593 %tmp8 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg2, i32 0, i32 0, i32 %tmp5 594 %tmp9 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg3, i32 0, i32 %tmp6, i32 0 595 %tmp10 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg4, i32 0, i32 0, i32 %tmp5 596 %tmp11 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg1, i32 0, i32 %tmp6, i32 1 597 %tmp12 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg2, i32 0, i32 1, i32 %tmp5 598 %tmp13 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg3, i32 0, i32 %tmp6, i32 1 599 %tmp14 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg4, i32 0, i32 1, i32 %tmp5 600 %tmp15 = load float, float addrspace(3)* %tmp7 601 %tmp16 = load float, float addrspace(3)* %tmp8 602 %tmp17 = fmul float %tmp15, %tmp16 603 %tmp18 = fadd float 2.000000e+00, %tmp17 604 %tmp19 = load float, float addrspace(3)* %tmp9 605 %tmp20 = load float, float addrspace(3)* %tmp10 606 %tmp21 = fmul float %tmp19, %tmp20 607 %tmp22 = fsub float %tmp18, %tmp21 608 %tmp23 = load float, float addrspace(3)* %tmp11 609 %tmp24 = load float, float addrspace(3)* %tmp12 610 %tmp25 = fmul float %tmp23, %tmp24 611 %tmp26 = fsub float %tmp22, %tmp25 612 %tmp27 = load float, float addrspace(3)* %tmp13 613 %tmp28 = load float, float addrspace(3)* %tmp14 614 %tmp29 = fmul float %tmp27, %tmp28 615 %tmp30 = fsub float %tmp26, %tmp29 616 store float %tmp30, float addrspace(1)* %tmp 617 ret void 618} 619 620; GCN-LABEL: ds_read_call_read: 621; GCN: ds_read_b32 622; GCN: s_swappc_b64 623; GCN: ds_read_b32 624define amdgpu_kernel void @ds_read_call_read(i32 addrspace(1)* %out, i32 addrspace(3)* %arg) { 625 %x = call i32 @llvm.amdgcn.workitem.id.x() 626 %arrayidx0 = getelementptr i32, i32 addrspace(3)* %arg, i32 %x 627 %arrayidx1 = getelementptr i32, i32 addrspace(3)* %arrayidx0, i32 1 628 %v0 = load i32, i32 addrspace(3)* %arrayidx0, align 4 629 call void @void_func_void() 630 %v1 = load i32, i32 addrspace(3)* %arrayidx1, align 4 631 %r = add i32 %v0, %v1 632 store i32 %r, i32 addrspace(1)* %out, align 4 633 ret void 634} 635 636; GCN-LABEL: {{^}}ds_read_interp_read: 637; CI: s_mov_b32 m0, -1 638; CI: ds_read_b32 639; CI: s_mov_b32 m0, s0 640; CI: v_interp_mov_f32 641; CI: s_mov_b32 m0, -1 642; CI: ds_read_b32 643; GFX9: ds_read2_b32 v[0:1], v0 offset1:4 644; GFX9: s_mov_b32 m0, s0 645; GFX9: v_interp_mov_f32 646define amdgpu_ps <2 x float> @ds_read_interp_read(i32 inreg %prims, float addrspace(3)* %inptr) { 647 %v0 = load float, float addrspace(3)* %inptr, align 4 648 %intrp = call float @llvm.amdgcn.interp.mov(i32 0, i32 0, i32 0, i32 %prims) 649 %ptr1 = getelementptr float, float addrspace(3)* %inptr, i32 4 650 %v1 = load float, float addrspace(3)* %ptr1, align 4 651 %v1b = fadd float %v1, %intrp 652 %r0 = insertelement <2 x float> undef, float %v0, i32 0 653 %r1 = insertelement <2 x float> %r0, float %v1b, i32 1 654 ret <2 x float> %r1 655} 656 657declare void @void_func_void() #3 658 659declare i32 @llvm.amdgcn.workgroup.id.x() #1 660declare i32 @llvm.amdgcn.workgroup.id.y() #1 661declare i32 @llvm.amdgcn.workitem.id.x() #1 662declare i32 @llvm.amdgcn.workitem.id.y() #1 663 664declare float @llvm.amdgcn.interp.mov(i32, i32, i32, i32) nounwind readnone 665 666declare void @llvm.amdgcn.s.barrier() #2 667 668attributes #0 = { nounwind } 669attributes #1 = { nounwind readnone speculatable } 670attributes #2 = { convergent nounwind } 671attributes #3 = { nounwind noinline } 672