1; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,CI %s 2; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=+load-store-opt,+flat-for-global,-unaligned-access-mode < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,GFX9,GFX9-ALIGNED %s 3; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=+load-store-opt,+flat-for-global,+unaligned-access-mode < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,GFX9,GFX9-UNALIGNED %s 4 5; FIXME: We don't get cases where the address was an SGPR because we 6; get a copy to the address register for each one. 7 8@lds = addrspace(3) global [512 x float] undef, align 4 9@lds.f64 = addrspace(3) global [512 x double] undef, align 8 10 11; GCN-LABEL: {{^}}simple_read2_f32: 12; CI-DAG: s_mov_b32 m0 13; GFX9-NOT: m0 14 15; GCN: ds_read2_b32 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:8 16; GCN: s_waitcnt lgkmcnt(0) 17; GCN: v_add_f32_e32 [[RESULT:v[0-9]+]], v[[LO_VREG]], v[[HI_VREG]] 18; CI: buffer_store_dword [[RESULT]] 19; GFX9: global_store_dword v{{[0-9]+}}, [[RESULT]], s{{\[[0-9]+:[0-9]+\]}} 20; GCN: s_endpgm 21define amdgpu_kernel void @simple_read2_f32(float addrspace(1)* %out) #0 { 22 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 23 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i 24 %val0 = load float, float addrspace(3)* %arrayidx0, align 4 25 %add.x = add nsw i32 %x.i, 8 26 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %add.x 27 %val1 = load float, float addrspace(3)* %arrayidx1, align 4 28 %sum = fadd float %val0, %val1 29 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 30 store float %sum, float addrspace(1)* %out.gep, align 4 31 ret void 32} 33 34; GCN-LABEL: {{^}}simple_read2_f32_max_offset: 35; CI-DAG: s_mov_b32 m0 36; GFX9-NOT: m0 37 38; GCN: ds_read2_b32 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:255 39; GCN: s_waitcnt lgkmcnt(0) 40; GCN: v_add_f32_e32 [[RESULT:v[0-9]+]], v[[LO_VREG]], v[[HI_VREG]] 41 42; CI: buffer_store_dword [[RESULT]] 43; GFX9: global_store_dword v{{[0-9]+}}, [[RESULT]], s{{\[[0-9]+:[0-9]+\]}} 44define amdgpu_kernel void @simple_read2_f32_max_offset(float addrspace(1)* %out) #0 { 45 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 46 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i 47 %val0 = load float, float addrspace(3)* %arrayidx0, align 4 48 %add.x = add nsw i32 %x.i, 255 49 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %add.x 50 %val1 = load float, float addrspace(3)* %arrayidx1, align 4 51 %sum = fadd float %val0, %val1 52 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 53 store float %sum, float addrspace(1)* %out.gep, align 4 54 ret void 55} 56 57; GCN-LABEL: @simple_read2_f32_too_far 58; CI-DAG: s_mov_b32 m0 59; GFX9-NOT: m0 60 61; GCN-NOT: ds_read2_b32 62; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} 63; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:1028 64; GCN: s_endpgm 65define amdgpu_kernel void @simple_read2_f32_too_far(float addrspace(1)* %out) #0 { 66 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 67 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i 68 %val0 = load float, float addrspace(3)* %arrayidx0, align 4 69 %add.x = add nsw i32 %x.i, 257 70 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %add.x 71 %val1 = load float, float addrspace(3)* %arrayidx1, align 4 72 %sum = fadd float %val0, %val1 73 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 74 store float %sum, float addrspace(1)* %out.gep, align 4 75 ret void 76} 77 78; GCN-LABEL: @simple_read2_f32_x2 79; CI-DAG: s_mov_b32 m0 80; GFX9-NOT: m0 81 82; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR:v[0-9]+]] offset1:8 83; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR]] offset0:11 offset1:27 84; GCN: s_endpgm 85define amdgpu_kernel void @simple_read2_f32_x2(float addrspace(1)* %out) #0 { 86 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 87 %idx.0 = add nsw i32 %tid.x, 0 88 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.0 89 %val0 = load float, float addrspace(3)* %arrayidx0, align 4 90 91 %idx.1 = add nsw i32 %tid.x, 8 92 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.1 93 %val1 = load float, float addrspace(3)* %arrayidx1, align 4 94 %sum.0 = fadd float %val0, %val1 95 96 %idx.2 = add nsw i32 %tid.x, 11 97 %arrayidx2 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.2 98 %val2 = load float, float addrspace(3)* %arrayidx2, align 4 99 100 %idx.3 = add nsw i32 %tid.x, 27 101 %arrayidx3 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.3 102 %val3 = load float, float addrspace(3)* %arrayidx3, align 4 103 %sum.1 = fadd float %val2, %val3 104 105 %sum = fadd float %sum.0, %sum.1 106 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %idx.0 107 store float %sum, float addrspace(1)* %out.gep, align 4 108 ret void 109} 110 111; Make sure there is an instruction between the two sets of reads. 112; GCN-LABEL: @simple_read2_f32_x2_barrier 113; CI-DAG: s_mov_b32 m0 114; GFX9-NOT: m0 115 116; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR:v[0-9]+]] offset1:8 117; GCN: s_barrier 118; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR]] offset0:11 offset1:27 119; GCN: s_endpgm 120define amdgpu_kernel void @simple_read2_f32_x2_barrier(float addrspace(1)* %out) #0 { 121 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 122 %idx.0 = add nsw i32 %tid.x, 0 123 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.0 124 %val0 = load float, float addrspace(3)* %arrayidx0, align 4 125 126 %idx.1 = add nsw i32 %tid.x, 8 127 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.1 128 %val1 = load float, float addrspace(3)* %arrayidx1, align 4 129 %sum.0 = fadd float %val0, %val1 130 131 call void @llvm.amdgcn.s.barrier() #2 132 133 %idx.2 = add nsw i32 %tid.x, 11 134 %arrayidx2 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.2 135 %val2 = load float, float addrspace(3)* %arrayidx2, align 4 136 137 %idx.3 = add nsw i32 %tid.x, 27 138 %arrayidx3 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.3 139 %val3 = load float, float addrspace(3)* %arrayidx3, align 4 140 %sum.1 = fadd float %val2, %val3 141 142 %sum = fadd float %sum.0, %sum.1 143 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %idx.0 144 store float %sum, float addrspace(1)* %out.gep, align 4 145 ret void 146} 147 148; For some reason adding something to the base address for the first 149; element results in only folding the inner pair. 150 151; GCN-LABEL: @simple_read2_f32_x2_nonzero_base 152; CI-DAG: s_mov_b32 m0 153; GFX9-NOT: m0 154 155; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR:v[0-9]+]] offset0:2 offset1:8 156; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR]] offset0:11 offset1:27 157; GCN: s_endpgm 158define amdgpu_kernel void @simple_read2_f32_x2_nonzero_base(float addrspace(1)* %out) #0 { 159 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 160 %idx.0 = add nsw i32 %tid.x, 2 161 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.0 162 %val0 = load float, float addrspace(3)* %arrayidx0, align 4 163 164 %idx.1 = add nsw i32 %tid.x, 8 165 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.1 166 %val1 = load float, float addrspace(3)* %arrayidx1, align 4 167 %sum.0 = fadd float %val0, %val1 168 169 %idx.2 = add nsw i32 %tid.x, 11 170 %arrayidx2 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.2 171 %val2 = load float, float addrspace(3)* %arrayidx2, align 4 172 173 %idx.3 = add nsw i32 %tid.x, 27 174 %arrayidx3 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.3 175 %val3 = load float, float addrspace(3)* %arrayidx3, align 4 176 %sum.1 = fadd float %val2, %val3 177 178 %sum = fadd float %sum.0, %sum.1 179 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %idx.0 180 store float %sum, float addrspace(1)* %out.gep, align 4 181 ret void 182} 183 184; Be careful of vectors of pointers. We don't know if the 2 pointers 185; in the vectors are really the same base, so this is not safe to 186; merge. 187; Base pointers come from different subregister of same super 188; register. We can't safely merge this. 189 190; GCN-LABEL: @read2_ptr_is_subreg_arg_f32 191; CI-DAG: s_mov_b32 m0 192; GFX9-NOT: m0 193 194; GCN-NOT: ds_read2_b32 195; GCN: ds_read_b32 196; GCN: ds_read_b32 197; GCN: s_endpgm 198define amdgpu_kernel void @read2_ptr_is_subreg_arg_f32(float addrspace(1)* %out, <2 x float addrspace(3)*> %lds.ptr) #0 { 199 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 200 %index.0 = insertelement <2 x i32> undef, i32 %x.i, i32 0 201 %index.1 = insertelement <2 x i32> %index.0, i32 8, i32 0 202 %gep = getelementptr inbounds float, <2 x float addrspace(3)*> %lds.ptr, <2 x i32> %index.1 203 %gep.0 = extractelement <2 x float addrspace(3)*> %gep, i32 0 204 %gep.1 = extractelement <2 x float addrspace(3)*> %gep, i32 1 205 %val0 = load float, float addrspace(3)* %gep.0, align 4 206 %val1 = load float, float addrspace(3)* %gep.1, align 4 207 %add.x = add nsw i32 %x.i, 8 208 %sum = fadd float %val0, %val1 209 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 210 store float %sum, float addrspace(1)* %out.gep, align 4 211 ret void 212} 213 214; Apply a constant scalar offset after the pointer vector extract. We 215; are rejecting merges that have the same, constant 0 offset, so make 216; sure we are really rejecting it because of the different 217; subregisters. 218 219; GCN-LABEL: @read2_ptr_is_subreg_arg_offset_f32 220; CI-DAG: s_mov_b32 m0 221; GFX9-NOT: m0 222 223; GCN-NOT: ds_read2_b32 224; GCN: ds_read_b32 225; GCN: ds_read_b32 226; GCN: s_endpgm 227define amdgpu_kernel void @read2_ptr_is_subreg_arg_offset_f32(float addrspace(1)* %out, <2 x float addrspace(3)*> %lds.ptr) #0 { 228 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 229 %index.0 = insertelement <2 x i32> undef, i32 %x.i, i32 0 230 %index.1 = insertelement <2 x i32> %index.0, i32 8, i32 0 231 %gep = getelementptr inbounds float, <2 x float addrspace(3)*> %lds.ptr, <2 x i32> %index.1 232 %gep.0 = extractelement <2 x float addrspace(3)*> %gep, i32 0 233 %gep.1 = extractelement <2 x float addrspace(3)*> %gep, i32 1 234 235 ; Apply an additional offset after the vector that will be more obviously folded. 236 %gep.1.offset = getelementptr float, float addrspace(3)* %gep.1, i32 8 237 238 %val0 = load float, float addrspace(3)* %gep.0, align 4 239 %val1 = load float, float addrspace(3)* %gep.1.offset, align 4 240 %add.x = add nsw i32 %x.i, 8 241 %sum = fadd float %val0, %val1 242 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 243 store float %sum, float addrspace(1)* %out.gep, align 4 244 ret void 245} 246 247; GCN-LABEL: {{^}}read2_ptr_is_subreg_f32: 248; CI-DAG: s_mov_b32 m0 249; GFX9-NOT: m0 250 251; GCN: ds_read2_b32 {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset1:8{{$}} 252; GCN: s_endpgm 253define amdgpu_kernel void @read2_ptr_is_subreg_f32(float addrspace(1)* %out) #0 { 254 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 255 %ptr.0 = insertelement <2 x [512 x float] addrspace(3)*> undef, [512 x float] addrspace(3)* @lds, i32 0 256 %ptr.1 = insertelement <2 x [512 x float] addrspace(3)*> %ptr.0, [512 x float] addrspace(3)* @lds, i32 1 257 %x.i.v.0 = insertelement <2 x i32> undef, i32 %x.i, i32 0 258 %x.i.v.1 = insertelement <2 x i32> %x.i.v.0, i32 %x.i, i32 1 259 %idx = add <2 x i32> %x.i.v.1, <i32 0, i32 8> 260 %gep = getelementptr inbounds [512 x float], <2 x [512 x float] addrspace(3)*> %ptr.1, <2 x i32> <i32 0, i32 0>, <2 x i32> %idx 261 %gep.0 = extractelement <2 x float addrspace(3)*> %gep, i32 0 262 %gep.1 = extractelement <2 x float addrspace(3)*> %gep, i32 1 263 %val0 = load float, float addrspace(3)* %gep.0, align 4 264 %val1 = load float, float addrspace(3)* %gep.1, align 4 265 %add.x = add nsw i32 %x.i, 8 266 %sum = fadd float %val0, %val1 267 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 268 store float %sum, float addrspace(1)* %out.gep, align 4 269 ret void 270} 271 272; GCN-LABEL: @simple_read2_f32_volatile_0 273; CI-DAG: s_mov_b32 m0 274; GFX9-NOT: m0 275 276; GCN-NOT: ds_read2_b32 277; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} 278; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:32 279; GCN: s_endpgm 280define amdgpu_kernel void @simple_read2_f32_volatile_0(float addrspace(1)* %out) #0 { 281 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 282 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i 283 %val0 = load volatile float, float addrspace(3)* %arrayidx0, align 4 284 %add.x = add nsw i32 %x.i, 8 285 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %add.x 286 %val1 = load float, float addrspace(3)* %arrayidx1, align 4 287 %sum = fadd float %val0, %val1 288 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 289 store float %sum, float addrspace(1)* %out.gep, align 4 290 ret void 291} 292 293; GCN-LABEL: @simple_read2_f32_volatile_1 294; CI-DAG: s_mov_b32 m0 295; GFX9-NOT: m0 296 297; GCN-NOT: ds_read2_b32 298; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} 299; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:32 300; GCN: s_endpgm 301define amdgpu_kernel void @simple_read2_f32_volatile_1(float addrspace(1)* %out) #0 { 302 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 303 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i 304 %val0 = load float, float addrspace(3)* %arrayidx0, align 4 305 %add.x = add nsw i32 %x.i, 8 306 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %add.x 307 %val1 = load volatile float, float addrspace(3)* %arrayidx1, align 4 308 %sum = fadd float %val0, %val1 309 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 310 store float %sum, float addrspace(1)* %out.gep, align 4 311 ret void 312} 313 314; Can't fold since not correctly aligned. 315; XXX: This isn't really testing anything useful now. I think CI 316; allows unaligned LDS accesses, which would be a problem here. 317; GCN-LABEL: @unaligned_read2_f32 318; CI-DAG: s_mov_b32 m0 319; GFX9-NOT: m0 320 321; CI-COUNT-4: ds_read_u8 322; GFX9-ALIGNED-4: ds_read_u8 323; GFX9-UNALIGNED-4: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}} 324; GCN: s_endpgm 325define amdgpu_kernel void @unaligned_read2_f32(float addrspace(1)* %out, float addrspace(3)* %lds) #0 { 326 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 327 %arrayidx0 = getelementptr inbounds float, float addrspace(3)* %lds, i32 %x.i 328 %val0 = load float, float addrspace(3)* %arrayidx0, align 1 329 %add.x = add nsw i32 %x.i, 8 330 %arrayidx1 = getelementptr inbounds float, float addrspace(3)* %lds, i32 %add.x 331 %val1 = load float, float addrspace(3)* %arrayidx1, align 1 332 %sum = fadd float %val0, %val1 333 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 334 store float %sum, float addrspace(1)* %out.gep, align 4 335 ret void 336} 337 338; GCN-LABEL: @misaligned_2_simple_read2_f32 339; CI-DAG: s_mov_b32 m0 340; GFX9-NOT: m0 341 342; CI-COUNT-2: ds_read_u16 343; GFX9-ALIGNED-2: ds_read_u16 344; GFX9-UNALIGNED-4: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}} 345; GCN: s_endpgm 346define amdgpu_kernel void @misaligned_2_simple_read2_f32(float addrspace(1)* %out, float addrspace(3)* %lds) #0 { 347 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 348 %arrayidx0 = getelementptr inbounds float, float addrspace(3)* %lds, i32 %x.i 349 %val0 = load float, float addrspace(3)* %arrayidx0, align 2 350 %add.x = add nsw i32 %x.i, 8 351 %arrayidx1 = getelementptr inbounds float, float addrspace(3)* %lds, i32 %add.x 352 %val1 = load float, float addrspace(3)* %arrayidx1, align 2 353 %sum = fadd float %val0, %val1 354 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 355 store float %sum, float addrspace(1)* %out.gep, align 4 356 ret void 357} 358 359; GCN-LABEL: @simple_read2_f64 360; CI-DAG: s_mov_b32 m0 361; GFX9-NOT: m0 362 363; GCN-DAG: v_lshlrev_b32_e32 [[VOFS:v[0-9]+]], 3, {{v[0-9]+}} 364; GCN-DAG: v_add_{{[iu]}}32_e32 [[VPTR:v[0-9]+]], {{(vcc, )?}}lds.f64@abs32@lo, [[VOFS]] 365; GCN: ds_read2_b64 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, [[VPTR]] offset1:8 366; GCN: v_add_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], v{{\[}}[[LO_VREG]]:{{[0-9]+\]}}, v{{\[[0-9]+}}:[[HI_VREG]]{{\]}} 367 368; CI: buffer_store_dwordx2 [[RESULT]] 369; GFX9: global_store_dwordx2 v{{[0-9]+}}, [[RESULT]], s{{\[[0-9]+:[0-9]+\]}} 370define amdgpu_kernel void @simple_read2_f64(double addrspace(1)* %out) #0 { 371 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 372 %arrayidx0 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %x.i 373 %val0 = load double, double addrspace(3)* %arrayidx0, align 8 374 %add.x = add nsw i32 %x.i, 8 375 %arrayidx1 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %add.x 376 %val1 = load double, double addrspace(3)* %arrayidx1, align 8 377 %sum = fadd double %val0, %val1 378 %out.gep = getelementptr inbounds double, double addrspace(1)* %out, i32 %x.i 379 store double %sum, double addrspace(1)* %out.gep, align 8 380 ret void 381} 382 383; GCN-LABEL: @simple_read2_f64_max_offset 384; CI-DAG: s_mov_b32 m0 385; GFX9-NOT: m0 386 387; GCN: ds_read2_b64 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:255 388; GCN: s_endpgm 389define amdgpu_kernel void @simple_read2_f64_max_offset(double addrspace(1)* %out) #0 { 390 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 391 %arrayidx0 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %x.i 392 %val0 = load double, double addrspace(3)* %arrayidx0, align 8 393 %add.x = add nsw i32 %x.i, 255 394 %arrayidx1 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %add.x 395 %val1 = load double, double addrspace(3)* %arrayidx1, align 8 396 %sum = fadd double %val0, %val1 397 %out.gep = getelementptr inbounds double, double addrspace(1)* %out, i32 %x.i 398 store double %sum, double addrspace(1)* %out.gep, align 8 399 ret void 400} 401 402; GCN-LABEL: @simple_read2_f64_too_far 403; CI-DAG: s_mov_b32 m0 404; GFX9-NOT: m0 405 406; GCN-NOT: ds_read2_b64 407; GCN: ds_read_b64 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} 408; GCN: ds_read_b64 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:2056 409; GCN: s_endpgm 410define amdgpu_kernel void @simple_read2_f64_too_far(double addrspace(1)* %out) #0 { 411 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 412 %arrayidx0 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %x.i 413 %val0 = load double, double addrspace(3)* %arrayidx0, align 8 414 %add.x = add nsw i32 %x.i, 257 415 %arrayidx1 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %add.x 416 %val1 = load double, double addrspace(3)* %arrayidx1, align 8 417 %sum = fadd double %val0, %val1 418 %out.gep = getelementptr inbounds double, double addrspace(1)* %out, i32 %x.i 419 store double %sum, double addrspace(1)* %out.gep, align 8 420 ret void 421} 422 423; Alignment only 4 424; GCN-LABEL: @misaligned_read2_f64 425; CI-DAG: s_mov_b32 m0 426; GFX9-NOT: m0 427 428; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset1:1 429; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset0:14 offset1:15 430; GCN: s_endpgm 431define amdgpu_kernel void @misaligned_read2_f64(double addrspace(1)* %out, double addrspace(3)* %lds) #0 { 432 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 433 %arrayidx0 = getelementptr inbounds double, double addrspace(3)* %lds, i32 %x.i 434 %val0 = load double, double addrspace(3)* %arrayidx0, align 4 435 %add.x = add nsw i32 %x.i, 7 436 %arrayidx1 = getelementptr inbounds double, double addrspace(3)* %lds, i32 %add.x 437 %val1 = load double, double addrspace(3)* %arrayidx1, align 4 438 %sum = fadd double %val0, %val1 439 %out.gep = getelementptr inbounds double, double addrspace(1)* %out, i32 %x.i 440 store double %sum, double addrspace(1)* %out.gep, align 4 441 ret void 442} 443 444@foo = addrspace(3) global [4 x i32] undef, align 4 445 446; GCN-LABEL: @load_constant_adjacent_offsets 447; CI-DAG: s_mov_b32 m0 448; GFX9-NOT: m0 449 450; GCN-DAG: v_mov_b32_e32 [[PTR:v[0-9]+]], foo@abs32@lo{{$}} 451; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]] offset1:1 452define amdgpu_kernel void @load_constant_adjacent_offsets(i32 addrspace(1)* %out) { 453 %val0 = load i32, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 0), align 4 454 %val1 = load i32, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 1), align 4 455 %sum = add i32 %val0, %val1 456 store i32 %sum, i32 addrspace(1)* %out, align 4 457 ret void 458} 459 460; GCN-LABEL: @load_constant_disjoint_offsets 461; CI-DAG: s_mov_b32 m0 462; GFX9-NOT: m0 463 464; GCN-DAG: v_mov_b32_e32 [[PTR:v[0-9]+]], foo@abs32@lo{{$}} 465; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]] offset1:2 466define amdgpu_kernel void @load_constant_disjoint_offsets(i32 addrspace(1)* %out) { 467 %val0 = load i32, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 0), align 4 468 %val1 = load i32, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 2), align 4 469 %sum = add i32 %val0, %val1 470 store i32 %sum, i32 addrspace(1)* %out, align 4 471 ret void 472} 473 474@bar = addrspace(3) global [4 x i64] undef, align 4 475 476; GCN-LABEL: @load_misaligned64_constant_offsets 477; CI-DAG: s_mov_b32 m0 478; GFX9-NOT: m0 479 480; GCN-DAG: v_mov_b32_e32 [[PTR:v[0-9]+]], bar@abs32@lo{{$}} 481; CI: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]] offset1:1 482; CI: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]] offset0:2 offset1:3 483; GFX9: ds_read_b128 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]] 484define amdgpu_kernel void @load_misaligned64_constant_offsets(i64 addrspace(1)* %out) { 485 %val0 = load i64, i64 addrspace(3)* getelementptr inbounds ([4 x i64], [4 x i64] addrspace(3)* @bar, i32 0, i32 0), align 4 486 %val1 = load i64, i64 addrspace(3)* getelementptr inbounds ([4 x i64], [4 x i64] addrspace(3)* @bar, i32 0, i32 1), align 4 487 %sum = add i64 %val0, %val1 488 store i64 %sum, i64 addrspace(1)* %out, align 8 489 ret void 490} 491 492@bar.large = addrspace(3) global [4096 x i64] undef, align 4 493 494; GCN-LABEL: @load_misaligned64_constant_large_offsets 495; CI-DAG: s_mov_b32 m0 496; GFX9-NOT: m0 497 498; GCN-DAG: s_mov_b32 [[SBASE0:s[0-9]+]], bar.large@abs32@lo 499; GCN-DAG: s_add_i32 [[SBASE1:s[0-9]+]], [[SBASE0]], 0x4000{{$}} 500; GCN-DAG: s_addk_i32 [[SBASE0]], 0x7ff8{{$}} 501; GCN-DAG: v_mov_b32_e32 [[VBASE0:v[0-9]+]], [[SBASE0]] 502; GCN-DAG: v_mov_b32_e32 [[VBASE1:v[0-9]+]], [[SBASE1]] 503; GCN-DAG: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[VBASE0]] offset1:1 504; GCN-DAG: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[VBASE1]] offset1:1 505; GCN: s_endpgm 506define amdgpu_kernel void @load_misaligned64_constant_large_offsets(i64 addrspace(1)* %out) { 507 %val0 = load i64, i64 addrspace(3)* getelementptr inbounds ([4096 x i64], [4096 x i64] addrspace(3)* @bar.large, i32 0, i32 2048), align 4 508 %val1 = load i64, i64 addrspace(3)* getelementptr inbounds ([4096 x i64], [4096 x i64] addrspace(3)* @bar.large, i32 0, i32 4095), align 4 509 %sum = add i64 %val0, %val1 510 store i64 %sum, i64 addrspace(1)* %out, align 8 511 ret void 512} 513 514@sgemm.lA = internal unnamed_addr addrspace(3) global [264 x float] undef, align 4 515@sgemm.lB = internal unnamed_addr addrspace(3) global [776 x float] undef, align 4 516 517; GCN-LABEL: {{^}}sgemm_inner_loop_read2_sequence: 518; CI-DAG: s_mov_b32 m0 519; GFX9-NOT: m0 520 521define amdgpu_kernel void @sgemm_inner_loop_read2_sequence(float addrspace(1)* %C, i32 %lda, i32 %ldb) #0 { 522 %x.i = tail call i32 @llvm.amdgcn.workgroup.id.x() #1 523 %y.i = tail call i32 @llvm.amdgcn.workitem.id.y() #1 524 %arrayidx44 = getelementptr inbounds [264 x float], [264 x float] addrspace(3)* @sgemm.lA, i32 0, i32 %x.i 525 %tmp16 = load float, float addrspace(3)* %arrayidx44, align 4 526 %add47 = add nsw i32 %x.i, 1 527 %arrayidx48 = getelementptr inbounds [264 x float], [264 x float] addrspace(3)* @sgemm.lA, i32 0, i32 %add47 528 %tmp17 = load float, float addrspace(3)* %arrayidx48, align 4 529 %add51 = add nsw i32 %x.i, 16 530 %arrayidx52 = getelementptr inbounds [264 x float], [264 x float] addrspace(3)* @sgemm.lA, i32 0, i32 %add51 531 %tmp18 = load float, float addrspace(3)* %arrayidx52, align 4 532 %add55 = add nsw i32 %x.i, 17 533 %arrayidx56 = getelementptr inbounds [264 x float], [264 x float] addrspace(3)* @sgemm.lA, i32 0, i32 %add55 534 %tmp19 = load float, float addrspace(3)* %arrayidx56, align 4 535 %arrayidx60 = getelementptr inbounds [776 x float], [776 x float] addrspace(3)* @sgemm.lB, i32 0, i32 %y.i 536 %tmp20 = load float, float addrspace(3)* %arrayidx60, align 4 537 %add63 = add nsw i32 %y.i, 1 538 %arrayidx64 = getelementptr inbounds [776 x float], [776 x float] addrspace(3)* @sgemm.lB, i32 0, i32 %add63 539 %tmp21 = load float, float addrspace(3)* %arrayidx64, align 4 540 %add67 = add nsw i32 %y.i, 32 541 %arrayidx68 = getelementptr inbounds [776 x float], [776 x float] addrspace(3)* @sgemm.lB, i32 0, i32 %add67 542 %tmp22 = load float, float addrspace(3)* %arrayidx68, align 4 543 %add71 = add nsw i32 %y.i, 33 544 %arrayidx72 = getelementptr inbounds [776 x float], [776 x float] addrspace(3)* @sgemm.lB, i32 0, i32 %add71 545 %tmp23 = load float, float addrspace(3)* %arrayidx72, align 4 546 %add75 = add nsw i32 %y.i, 64 547 %arrayidx76 = getelementptr inbounds [776 x float], [776 x float] addrspace(3)* @sgemm.lB, i32 0, i32 %add75 548 %tmp24 = load float, float addrspace(3)* %arrayidx76, align 4 549 %add79 = add nsw i32 %y.i, 65 550 %arrayidx80 = getelementptr inbounds [776 x float], [776 x float] addrspace(3)* @sgemm.lB, i32 0, i32 %add79 551 %tmp25 = load float, float addrspace(3)* %arrayidx80, align 4 552 %sum.0 = fadd float %tmp16, %tmp17 553 %sum.1 = fadd float %sum.0, %tmp18 554 %sum.2 = fadd float %sum.1, %tmp19 555 %sum.3 = fadd float %sum.2, %tmp20 556 %sum.4 = fadd float %sum.3, %tmp21 557 %sum.5 = fadd float %sum.4, %tmp22 558 %sum.6 = fadd float %sum.5, %tmp23 559 %sum.7 = fadd float %sum.6, %tmp24 560 %sum.8 = fadd float %sum.7, %tmp25 561 store float %sum.8, float addrspace(1)* %C, align 4 562 ret void 563} 564 565; GCN-LABEL: {{^}}misaligned_read2_v2i32: 566; CI-DAG: s_mov_b32 m0 567; GFX9-NOT: m0 568define amdgpu_kernel void @misaligned_read2_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(3)* %in) #0 { 569 %load = load <2 x i32>, <2 x i32> addrspace(3)* %in, align 4 570 store <2 x i32> %load, <2 x i32> addrspace(1)* %out, align 8 571 ret void 572} 573 574; GCN-LABEL: {{^}}misaligned_read2_i64: 575; CI-DAG: s_mov_b32 m0 576; GFX9-NOT: m0 577define amdgpu_kernel void @misaligned_read2_i64(i64 addrspace(1)* %out, i64 addrspace(3)* %in) #0 { 578 %load = load i64, i64 addrspace(3)* %in, align 4 579 store i64 %load, i64 addrspace(1)* %out, align 8 580 ret void 581} 582 583; GCN-LABEL: ds_read_diff_base_interleaving 584; CI-DAG: s_mov_b32 m0 585; GFX9-NOT: m0 586 587; GCN-NOT: ds_read_b32 588define amdgpu_kernel void @ds_read_diff_base_interleaving( 589 float addrspace(1)* nocapture %arg, 590 [4 x [4 x float]] addrspace(3)* %arg1, 591 [4 x [4 x float]] addrspace(3)* %arg2, 592 [4 x [4 x float]] addrspace(3)* %arg3, 593 [4 x [4 x float]] addrspace(3)* %arg4) #1 { 594bb: 595 %tmp = getelementptr float, float addrspace(1)* %arg, i64 10 596 %tmp5 = tail call i32 @llvm.amdgcn.workitem.id.x() #2 597 %tmp6 = tail call i32 @llvm.amdgcn.workitem.id.y() #2 598 %tmp7 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg1, i32 0, i32 %tmp6, i32 0 599 %tmp8 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg2, i32 0, i32 0, i32 %tmp5 600 %tmp9 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg3, i32 0, i32 %tmp6, i32 0 601 %tmp10 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg4, i32 0, i32 0, i32 %tmp5 602 %tmp11 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg1, i32 0, i32 %tmp6, i32 1 603 %tmp12 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg2, i32 0, i32 1, i32 %tmp5 604 %tmp13 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg3, i32 0, i32 %tmp6, i32 1 605 %tmp14 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg4, i32 0, i32 1, i32 %tmp5 606 %tmp15 = load float, float addrspace(3)* %tmp7 607 %tmp16 = load float, float addrspace(3)* %tmp8 608 %tmp17 = fmul float %tmp15, %tmp16 609 %tmp18 = fadd float 2.000000e+00, %tmp17 610 %tmp19 = load float, float addrspace(3)* %tmp9 611 %tmp20 = load float, float addrspace(3)* %tmp10 612 %tmp21 = fmul float %tmp19, %tmp20 613 %tmp22 = fsub float %tmp18, %tmp21 614 %tmp23 = load float, float addrspace(3)* %tmp11 615 %tmp24 = load float, float addrspace(3)* %tmp12 616 %tmp25 = fmul float %tmp23, %tmp24 617 %tmp26 = fsub float %tmp22, %tmp25 618 %tmp27 = load float, float addrspace(3)* %tmp13 619 %tmp28 = load float, float addrspace(3)* %tmp14 620 %tmp29 = fmul float %tmp27, %tmp28 621 %tmp30 = fsub float %tmp26, %tmp29 622 store float %tmp30, float addrspace(1)* %tmp 623 ret void 624} 625 626; GCN-LABEL: ds_read_call_read: 627; GCN: ds_read_b32 628; GCN: s_swappc_b64 629; GCN: ds_read_b32 630define amdgpu_kernel void @ds_read_call_read(i32 addrspace(1)* %out, i32 addrspace(3)* %arg) { 631 %x = call i32 @llvm.amdgcn.workitem.id.x() 632 %arrayidx0 = getelementptr i32, i32 addrspace(3)* %arg, i32 %x 633 %arrayidx1 = getelementptr i32, i32 addrspace(3)* %arrayidx0, i32 1 634 %v0 = load i32, i32 addrspace(3)* %arrayidx0, align 4 635 call void @void_func_void() 636 %v1 = load i32, i32 addrspace(3)* %arrayidx1, align 4 637 %r = add i32 %v0, %v1 638 store i32 %r, i32 addrspace(1)* %out, align 4 639 ret void 640} 641 642; GCN-LABEL: {{^}}ds_read_interp_read: 643; CI: s_mov_b32 m0, -1 644; CI: ds_read_b32 645; CI: s_mov_b32 m0, s0 646; CI: v_interp_mov_f32 647; CI: s_mov_b32 m0, -1 648; CI: ds_read_b32 649; GFX9: ds_read2_b32 v[0:1], v0 offset1:4 650; GFX9: s_mov_b32 m0, s0 651; GFX9: v_interp_mov_f32 652define amdgpu_ps <2 x float> @ds_read_interp_read(i32 inreg %prims, float addrspace(3)* %inptr) { 653 %v0 = load float, float addrspace(3)* %inptr, align 4 654 %intrp = call float @llvm.amdgcn.interp.mov(i32 0, i32 0, i32 0, i32 %prims) 655 %ptr1 = getelementptr float, float addrspace(3)* %inptr, i32 4 656 %v1 = load float, float addrspace(3)* %ptr1, align 4 657 %v1b = fadd float %v1, %intrp 658 %r0 = insertelement <2 x float> undef, float %v0, i32 0 659 %r1 = insertelement <2 x float> %r0, float %v1b, i32 1 660 ret <2 x float> %r1 661} 662 663@v2i32_align1 = internal addrspace(3) global [100 x <2 x i32>] undef, align 1 664 665; GCN-LABEL: {{^}}read2_v2i32_align1_odd_offset: 666; CI-COUNT-8: ds_read_u8 667 668; GFX9-ALIGNED-COUNT-8: ds_read_u8 669 670; GFX9-UNALIGNED: v_mov_b32_e32 [[BASE_ADDR:v[0-9]+]], 0x41{{$}} 671; GFX9-UNALIGNED: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASE_ADDR]] offset1:1{{$}} 672define amdgpu_kernel void @read2_v2i32_align1_odd_offset(<2 x i32> addrspace(1)* %out) { 673entry: 674 %load = load <2 x i32>, <2 x i32> addrspace(3)* bitcast (i8 addrspace(3)* getelementptr (i8, i8 addrspace(3)* bitcast ([100 x <2 x i32>] addrspace(3)* @v2i32_align1 to i8 addrspace(3)*), i32 65) to <2 x i32> addrspace(3)*), align 1 675 store <2 x i32> %load, <2 x i32> addrspace(1)* %out 676 ret void 677} 678 679declare void @void_func_void() #3 680 681declare i32 @llvm.amdgcn.workgroup.id.x() #1 682declare i32 @llvm.amdgcn.workgroup.id.y() #1 683declare i32 @llvm.amdgcn.workitem.id.x() #1 684declare i32 @llvm.amdgcn.workitem.id.y() #1 685 686declare float @llvm.amdgcn.interp.mov(i32, i32, i32, i32) nounwind readnone 687 688declare void @llvm.amdgcn.s.barrier() #2 689 690attributes #0 = { nounwind } 691attributes #1 = { nounwind readnone speculatable } 692attributes #2 = { convergent nounwind } 693attributes #3 = { nounwind noinline } 694