1; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs -amdgpu-load-store-vectorizer=0 < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=GCN-AA %s 2; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=bonaire -verify-machineinstrs -amdgpu-load-store-vectorizer=0 < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=GCN-AA %s 3 4; This test is mostly to test DAG store merging, so disable the vectorizer. 5; Run with devices with different unaligned load restrictions. 6 7; TODO: Vector element tests 8; TODO: Non-zero base offset for load and store combinations 9; TODO: Same base addrspacecasted 10 11 12; GCN-LABEL: {{^}}merge_global_store_2_constants_i8: 13; GCN: buffer_store_short 14; GCN: s_endpgm 15define amdgpu_kernel void @merge_global_store_2_constants_i8(i8 addrspace(1)* %out) #0 { 16 %out.gep.1 = getelementptr i8, i8 addrspace(1)* %out, i32 1 17 18 store i8 123, i8 addrspace(1)* %out.gep.1 19 store i8 456, i8 addrspace(1)* %out, align 2 20 ret void 21} 22 23; GCN-LABEL: {{^}}merge_global_store_2_constants_i8_natural_align: 24; GCN: buffer_store_byte 25; GCN: buffer_store_byte 26; GCN: s_endpgm 27define amdgpu_kernel void @merge_global_store_2_constants_i8_natural_align(i8 addrspace(1)* %out) #0 { 28 %out.gep.1 = getelementptr i8, i8 addrspace(1)* %out, i32 1 29 30 store i8 123, i8 addrspace(1)* %out.gep.1 31 store i8 456, i8 addrspace(1)* %out 32 ret void 33} 34 35; GCN-LABEL: {{^}}merge_global_store_2_constants_i16: 36; GCN: buffer_store_dword v 37define amdgpu_kernel void @merge_global_store_2_constants_i16(i16 addrspace(1)* %out) #0 { 38 %out.gep.1 = getelementptr i16, i16 addrspace(1)* %out, i32 1 39 40 store i16 123, i16 addrspace(1)* %out.gep.1 41 store i16 456, i16 addrspace(1)* %out, align 4 42 ret void 43} 44 45; GCN-LABEL: {{^}}merge_global_store_2_constants_0_i16: 46; GCN: buffer_store_dword v 47define amdgpu_kernel void @merge_global_store_2_constants_0_i16(i16 addrspace(1)* %out) #0 { 48 %out.gep.1 = getelementptr i16, i16 addrspace(1)* %out, i32 1 49 50 store i16 0, i16 addrspace(1)* %out.gep.1 51 store i16 0, i16 addrspace(1)* %out, align 4 52 ret void 53} 54 55; GCN-LABEL: {{^}}merge_global_store_2_constants_i16_natural_align: 56; GCN: buffer_store_short 57; GCN: buffer_store_short 58; GCN: s_endpgm 59define amdgpu_kernel void @merge_global_store_2_constants_i16_natural_align(i16 addrspace(1)* %out) #0 { 60 %out.gep.1 = getelementptr i16, i16 addrspace(1)* %out, i32 1 61 62 store i16 123, i16 addrspace(1)* %out.gep.1 63 store i16 456, i16 addrspace(1)* %out 64 ret void 65} 66 67; GCN-LABEL: {{^}}merge_global_store_2_constants_i32: 68; SI-DAG: v_mov_b32_e32 v[[LO:[0-9]+]], 0x1c8 69; SI-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], 0x7b 70; GCN: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]{{\]}} 71define amdgpu_kernel void @merge_global_store_2_constants_i32(i32 addrspace(1)* %out) #0 { 72 %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1 73 74 store i32 123, i32 addrspace(1)* %out.gep.1 75 store i32 456, i32 addrspace(1)* %out 76 ret void 77} 78 79; GCN-LABEL: {{^}}merge_global_store_2_constants_i32_f32: 80; GCN: buffer_store_dwordx2 81define amdgpu_kernel void @merge_global_store_2_constants_i32_f32(i32 addrspace(1)* %out) #0 { 82 %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1 83 %out.gep.1.bc = bitcast i32 addrspace(1)* %out.gep.1 to float addrspace(1)* 84 store float 1.0, float addrspace(1)* %out.gep.1.bc 85 store i32 456, i32 addrspace(1)* %out 86 ret void 87} 88 89; GCN-LABEL: {{^}}merge_global_store_2_constants_f32_i32: 90; SI-DAG: v_mov_b32_e32 v[[VLO:[0-9]+]], 4.0 91; SI-DAG: v_mov_b32_e32 v[[VHI:[0-9]+]], 0x7b 92; GCN: buffer_store_dwordx2 v{{\[}}[[VLO]]:[[VHI]]{{\]}} 93define amdgpu_kernel void @merge_global_store_2_constants_f32_i32(float addrspace(1)* %out) #0 { 94 %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1 95 %out.gep.1.bc = bitcast float addrspace(1)* %out.gep.1 to i32 addrspace(1)* 96 store i32 123, i32 addrspace(1)* %out.gep.1.bc 97 store float 4.0, float addrspace(1)* %out 98 ret void 99} 100 101; GCN-LABEL: {{^}}merge_global_store_4_constants_i32: 102; GCN-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], 0x14d{{$}} 103; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x1c8{{$}} 104; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x7b{{$}} 105; GCN-DAG: v_mov_b32_e32 v[[LO:[0-9]+]], 0x4d2{{$}} 106; GCN: buffer_store_dwordx4 v{{\[}}[[LO]]:[[HI]]{{\]}} 107define amdgpu_kernel void @merge_global_store_4_constants_i32(i32 addrspace(1)* %out) #0 { 108 %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1 109 %out.gep.2 = getelementptr i32, i32 addrspace(1)* %out, i32 2 110 %out.gep.3 = getelementptr i32, i32 addrspace(1)* %out, i32 3 111 112 store i32 123, i32 addrspace(1)* %out.gep.1 113 store i32 456, i32 addrspace(1)* %out.gep.2 114 store i32 333, i32 addrspace(1)* %out.gep.3 115 store i32 1234, i32 addrspace(1)* %out 116 ret void 117} 118 119; GCN-LABEL: {{^}}merge_global_store_4_constants_f32_order: 120; GCN: buffer_store_dwordx4 121define amdgpu_kernel void @merge_global_store_4_constants_f32_order(float addrspace(1)* %out) #0 { 122 %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1 123 %out.gep.2 = getelementptr float, float addrspace(1)* %out, i32 2 124 %out.gep.3 = getelementptr float, float addrspace(1)* %out, i32 3 125 126 store float 8.0, float addrspace(1)* %out 127 store float 1.0, float addrspace(1)* %out.gep.1 128 store float 2.0, float addrspace(1)* %out.gep.2 129 store float 4.0, float addrspace(1)* %out.gep.3 130 ret void 131} 132 133; First store is out of order. 134; GCN-LABEL: {{^}}merge_global_store_4_constants_f32: 135; GCN: buffer_store_dwordx4 136define amdgpu_kernel void @merge_global_store_4_constants_f32(float addrspace(1)* %out) #0 { 137 %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1 138 %out.gep.2 = getelementptr float, float addrspace(1)* %out, i32 2 139 %out.gep.3 = getelementptr float, float addrspace(1)* %out, i32 3 140 141 store float 1.0, float addrspace(1)* %out.gep.1 142 store float 2.0, float addrspace(1)* %out.gep.2 143 store float 4.0, float addrspace(1)* %out.gep.3 144 store float 8.0, float addrspace(1)* %out 145 ret void 146} 147 148; GCN-LABEL: {{^}}merge_global_store_4_constants_mixed_i32_f32: 149; GCN-AA: buffer_store_dwordx4 v 150; GCN: s_endpgm 151define amdgpu_kernel void @merge_global_store_4_constants_mixed_i32_f32(float addrspace(1)* %out) #0 { 152 %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1 153 %out.gep.2 = getelementptr float, float addrspace(1)* %out, i32 2 154 %out.gep.3 = getelementptr float, float addrspace(1)* %out, i32 3 155 156 %out.gep.1.bc = bitcast float addrspace(1)* %out.gep.1 to i32 addrspace(1)* 157 %out.gep.3.bc = bitcast float addrspace(1)* %out.gep.3 to i32 addrspace(1)* 158 159 store i32 11, i32 addrspace(1)* %out.gep.1.bc 160 store float 2.0, float addrspace(1)* %out.gep.2 161 store i32 17, i32 addrspace(1)* %out.gep.3.bc 162 store float 8.0, float addrspace(1)* %out 163 ret void 164} 165 166; GCN-LABEL: {{^}}merge_global_store_3_constants_i32: 167; SI-DAG: buffer_store_dwordx3 168; SI-NOT: buffer_store_dwordx2 169; SI-NOT: buffer_store_dword 170; GCN: s_endpgm 171define amdgpu_kernel void @merge_global_store_3_constants_i32(i32 addrspace(1)* %out) #0 { 172 %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1 173 %out.gep.2 = getelementptr i32, i32 addrspace(1)* %out, i32 2 174 175 store i32 123, i32 addrspace(1)* %out.gep.1 176 store i32 456, i32 addrspace(1)* %out.gep.2 177 store i32 1234, i32 addrspace(1)* %out 178 ret void 179} 180 181; GCN-LABEL: {{^}}merge_global_store_2_constants_i64: 182; GCN: buffer_store_dwordx4 183define amdgpu_kernel void @merge_global_store_2_constants_i64(i64 addrspace(1)* %out) #0 { 184 %out.gep.1 = getelementptr i64, i64 addrspace(1)* %out, i64 1 185 186 store i64 123, i64 addrspace(1)* %out.gep.1 187 store i64 456, i64 addrspace(1)* %out 188 ret void 189} 190 191; GCN-LABEL: {{^}}merge_global_store_4_constants_i64: 192; GCN: buffer_store_dwordx4 193; GCN: buffer_store_dwordx4 194define amdgpu_kernel void @merge_global_store_4_constants_i64(i64 addrspace(1)* %out) #0 { 195 %out.gep.1 = getelementptr i64, i64 addrspace(1)* %out, i64 1 196 %out.gep.2 = getelementptr i64, i64 addrspace(1)* %out, i64 2 197 %out.gep.3 = getelementptr i64, i64 addrspace(1)* %out, i64 3 198 199 store i64 123, i64 addrspace(1)* %out.gep.1 200 store i64 456, i64 addrspace(1)* %out.gep.2 201 store i64 333, i64 addrspace(1)* %out.gep.3 202 store i64 1234, i64 addrspace(1)* %out 203 ret void 204} 205 206; GCN-LABEL: {{^}}merge_global_store_2_adjacent_loads_i32: 207; GCN: buffer_load_dwordx2 [[LOAD:v\[[0-9]+:[0-9]+\]]] 208; GCN: buffer_store_dwordx2 [[LOAD]] 209define amdgpu_kernel void @merge_global_store_2_adjacent_loads_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 { 210 %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1 211 %in.gep.1 = getelementptr i32, i32 addrspace(1)* %in, i32 1 212 213 %lo = load i32, i32 addrspace(1)* %in 214 %hi = load i32, i32 addrspace(1)* %in.gep.1 215 216 store i32 %lo, i32 addrspace(1)* %out 217 store i32 %hi, i32 addrspace(1)* %out.gep.1 218 ret void 219} 220 221; GCN-LABEL: {{^}}merge_global_store_2_adjacent_loads_i32_nonzero_base: 222; GCN: buffer_load_dwordx2 [[LOAD:v\[[0-9]+:[0-9]+\]]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:8 223; GCN: buffer_store_dwordx2 [[LOAD]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:8 224define amdgpu_kernel void @merge_global_store_2_adjacent_loads_i32_nonzero_base(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 { 225 %in.gep.0 = getelementptr i32, i32 addrspace(1)* %in, i32 2 226 %in.gep.1 = getelementptr i32, i32 addrspace(1)* %in, i32 3 227 228 %out.gep.0 = getelementptr i32, i32 addrspace(1)* %out, i32 2 229 %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 3 230 %lo = load i32, i32 addrspace(1)* %in.gep.0 231 %hi = load i32, i32 addrspace(1)* %in.gep.1 232 233 store i32 %lo, i32 addrspace(1)* %out.gep.0 234 store i32 %hi, i32 addrspace(1)* %out.gep.1 235 ret void 236} 237 238; GCN-LABEL: {{^}}merge_global_store_2_adjacent_loads_shuffle_i32: 239; GCN: buffer_load_dwordx2 v 240; GCN: buffer_store_dwordx2 v 241define amdgpu_kernel void @merge_global_store_2_adjacent_loads_shuffle_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 { 242 %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1 243 %in.gep.1 = getelementptr i32, i32 addrspace(1)* %in, i32 1 244 245 %lo = load i32, i32 addrspace(1)* %in 246 %hi = load i32, i32 addrspace(1)* %in.gep.1 247 248 store i32 %hi, i32 addrspace(1)* %out 249 store i32 %lo, i32 addrspace(1)* %out.gep.1 250 ret void 251} 252 253; GCN-LABEL: {{^}}merge_global_store_4_adjacent_loads_i32: 254; GCN: buffer_load_dwordx4 [[LOAD:v\[[0-9]+:[0-9]+\]]] 255; GCN: buffer_store_dwordx4 [[LOAD]] 256define amdgpu_kernel void @merge_global_store_4_adjacent_loads_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 { 257 %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1 258 %out.gep.2 = getelementptr i32, i32 addrspace(1)* %out, i32 2 259 %out.gep.3 = getelementptr i32, i32 addrspace(1)* %out, i32 3 260 %in.gep.1 = getelementptr i32, i32 addrspace(1)* %in, i32 1 261 %in.gep.2 = getelementptr i32, i32 addrspace(1)* %in, i32 2 262 %in.gep.3 = getelementptr i32, i32 addrspace(1)* %in, i32 3 263 264 %x = load i32, i32 addrspace(1)* %in 265 %y = load i32, i32 addrspace(1)* %in.gep.1 266 %z = load i32, i32 addrspace(1)* %in.gep.2 267 %w = load i32, i32 addrspace(1)* %in.gep.3 268 269 store i32 %x, i32 addrspace(1)* %out 270 store i32 %y, i32 addrspace(1)* %out.gep.1 271 store i32 %z, i32 addrspace(1)* %out.gep.2 272 store i32 %w, i32 addrspace(1)* %out.gep.3 273 ret void 274} 275 276; GCN-LABEL: {{^}}merge_global_store_3_adjacent_loads_i32: 277; SI-DAG: buffer_load_dwordx3 278; GCN: s_waitcnt 279; SI-DAG: buffer_store_dwordx3 v 280; GCN: s_endpgm 281define amdgpu_kernel void @merge_global_store_3_adjacent_loads_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 { 282 %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1 283 %out.gep.2 = getelementptr i32, i32 addrspace(1)* %out, i32 2 284 %in.gep.1 = getelementptr i32, i32 addrspace(1)* %in, i32 1 285 %in.gep.2 = getelementptr i32, i32 addrspace(1)* %in, i32 2 286 287 %x = load i32, i32 addrspace(1)* %in 288 %y = load i32, i32 addrspace(1)* %in.gep.1 289 %z = load i32, i32 addrspace(1)* %in.gep.2 290 291 store i32 %x, i32 addrspace(1)* %out 292 store i32 %y, i32 addrspace(1)* %out.gep.1 293 store i32 %z, i32 addrspace(1)* %out.gep.2 294 ret void 295} 296 297; GCN-LABEL: {{^}}merge_global_store_4_adjacent_loads_f32: 298; GCN: buffer_load_dwordx4 [[LOAD:v\[[0-9]+:[0-9]+\]]] 299; GCN: buffer_store_dwordx4 [[LOAD]] 300define amdgpu_kernel void @merge_global_store_4_adjacent_loads_f32(float addrspace(1)* %out, float addrspace(1)* %in) #0 { 301 %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1 302 %out.gep.2 = getelementptr float, float addrspace(1)* %out, i32 2 303 %out.gep.3 = getelementptr float, float addrspace(1)* %out, i32 3 304 %in.gep.1 = getelementptr float, float addrspace(1)* %in, i32 1 305 %in.gep.2 = getelementptr float, float addrspace(1)* %in, i32 2 306 %in.gep.3 = getelementptr float, float addrspace(1)* %in, i32 3 307 308 %x = load float, float addrspace(1)* %in 309 %y = load float, float addrspace(1)* %in.gep.1 310 %z = load float, float addrspace(1)* %in.gep.2 311 %w = load float, float addrspace(1)* %in.gep.3 312 313 store float %x, float addrspace(1)* %out 314 store float %y, float addrspace(1)* %out.gep.1 315 store float %z, float addrspace(1)* %out.gep.2 316 store float %w, float addrspace(1)* %out.gep.3 317 ret void 318} 319 320; GCN-LABEL: {{^}}merge_global_store_4_adjacent_loads_i32_nonzero_base: 321; GCN: buffer_load_dwordx4 [[LOAD:v\[[0-9]+:[0-9]+\]]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:44 322; GCN: buffer_store_dwordx4 [[LOAD]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:28 323define amdgpu_kernel void @merge_global_store_4_adjacent_loads_i32_nonzero_base(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 { 324 %in.gep.0 = getelementptr i32, i32 addrspace(1)* %in, i32 11 325 %in.gep.1 = getelementptr i32, i32 addrspace(1)* %in, i32 12 326 %in.gep.2 = getelementptr i32, i32 addrspace(1)* %in, i32 13 327 %in.gep.3 = getelementptr i32, i32 addrspace(1)* %in, i32 14 328 %out.gep.0 = getelementptr i32, i32 addrspace(1)* %out, i32 7 329 %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 8 330 %out.gep.2 = getelementptr i32, i32 addrspace(1)* %out, i32 9 331 %out.gep.3 = getelementptr i32, i32 addrspace(1)* %out, i32 10 332 333 %x = load i32, i32 addrspace(1)* %in.gep.0 334 %y = load i32, i32 addrspace(1)* %in.gep.1 335 %z = load i32, i32 addrspace(1)* %in.gep.2 336 %w = load i32, i32 addrspace(1)* %in.gep.3 337 338 store i32 %x, i32 addrspace(1)* %out.gep.0 339 store i32 %y, i32 addrspace(1)* %out.gep.1 340 store i32 %z, i32 addrspace(1)* %out.gep.2 341 store i32 %w, i32 addrspace(1)* %out.gep.3 342 ret void 343} 344 345; GCN-LABEL: {{^}}merge_global_store_4_adjacent_loads_inverse_i32: 346; GCN: buffer_load_dwordx4 [[LOAD:v\[[0-9]+:[0-9]+\]]] 347; GCN: s_barrier 348; GCN: buffer_store_dwordx4 [[LOAD]] 349define amdgpu_kernel void @merge_global_store_4_adjacent_loads_inverse_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 { 350 %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1 351 %out.gep.2 = getelementptr i32, i32 addrspace(1)* %out, i32 2 352 %out.gep.3 = getelementptr i32, i32 addrspace(1)* %out, i32 3 353 %in.gep.1 = getelementptr i32, i32 addrspace(1)* %in, i32 1 354 %in.gep.2 = getelementptr i32, i32 addrspace(1)* %in, i32 2 355 %in.gep.3 = getelementptr i32, i32 addrspace(1)* %in, i32 3 356 357 %x = load i32, i32 addrspace(1)* %in 358 %y = load i32, i32 addrspace(1)* %in.gep.1 359 %z = load i32, i32 addrspace(1)* %in.gep.2 360 %w = load i32, i32 addrspace(1)* %in.gep.3 361 362 ; Make sure the barrier doesn't stop this 363 tail call void @llvm.amdgcn.s.barrier() #1 364 365 store i32 %w, i32 addrspace(1)* %out.gep.3 366 store i32 %z, i32 addrspace(1)* %out.gep.2 367 store i32 %y, i32 addrspace(1)* %out.gep.1 368 store i32 %x, i32 addrspace(1)* %out 369 370 ret void 371} 372 373; TODO: Re-packing of loaded register required. Maybe an IR pass 374; should catch this? 375 376; GCN-LABEL: {{^}}merge_global_store_4_adjacent_loads_shuffle_i32: 377; GCN: buffer_load_dwordx4 v 378; GCN: s_barrier 379; GCN: buffer_store_dwordx4 v 380define amdgpu_kernel void @merge_global_store_4_adjacent_loads_shuffle_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 { 381 %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1 382 %out.gep.2 = getelementptr i32, i32 addrspace(1)* %out, i32 2 383 %out.gep.3 = getelementptr i32, i32 addrspace(1)* %out, i32 3 384 %in.gep.1 = getelementptr i32, i32 addrspace(1)* %in, i32 1 385 %in.gep.2 = getelementptr i32, i32 addrspace(1)* %in, i32 2 386 %in.gep.3 = getelementptr i32, i32 addrspace(1)* %in, i32 3 387 388 %x = load i32, i32 addrspace(1)* %in 389 %y = load i32, i32 addrspace(1)* %in.gep.1 390 %z = load i32, i32 addrspace(1)* %in.gep.2 391 %w = load i32, i32 addrspace(1)* %in.gep.3 392 393 ; Make sure the barrier doesn't stop this 394 tail call void @llvm.amdgcn.s.barrier() #1 395 396 store i32 %w, i32 addrspace(1)* %out 397 store i32 %z, i32 addrspace(1)* %out.gep.1 398 store i32 %y, i32 addrspace(1)* %out.gep.2 399 store i32 %x, i32 addrspace(1)* %out.gep.3 400 401 ret void 402} 403 404; GCN-LABEL: {{^}}merge_global_store_4_adjacent_loads_i8: 405; GCN: buffer_load_dword [[LOAD:v[0-9]+]] 406; GCN: buffer_store_dword [[LOAD]] 407; GCN: s_endpgm 408define amdgpu_kernel void @merge_global_store_4_adjacent_loads_i8(i8 addrspace(1)* %out, i8 addrspace(1)* %in) #0 { 409 %out.gep.1 = getelementptr i8, i8 addrspace(1)* %out, i8 1 410 %out.gep.2 = getelementptr i8, i8 addrspace(1)* %out, i8 2 411 %out.gep.3 = getelementptr i8, i8 addrspace(1)* %out, i8 3 412 %in.gep.1 = getelementptr i8, i8 addrspace(1)* %in, i8 1 413 %in.gep.2 = getelementptr i8, i8 addrspace(1)* %in, i8 2 414 %in.gep.3 = getelementptr i8, i8 addrspace(1)* %in, i8 3 415 416 %x = load i8, i8 addrspace(1)* %in, align 4 417 %y = load i8, i8 addrspace(1)* %in.gep.1 418 %z = load i8, i8 addrspace(1)* %in.gep.2 419 %w = load i8, i8 addrspace(1)* %in.gep.3 420 421 store i8 %x, i8 addrspace(1)* %out, align 4 422 store i8 %y, i8 addrspace(1)* %out.gep.1 423 store i8 %z, i8 addrspace(1)* %out.gep.2 424 store i8 %w, i8 addrspace(1)* %out.gep.3 425 ret void 426} 427 428; GCN-LABEL: {{^}}merge_global_store_4_adjacent_loads_i8_natural_align: 429; GCN: buffer_load_ubyte 430; GCN: buffer_load_ubyte 431; GCN: buffer_load_ubyte 432; GCN: buffer_load_ubyte 433; GCN: buffer_store_byte 434; GCN: buffer_store_byte 435; GCN: buffer_store_byte 436; GCN: buffer_store_byte 437; GCN: s_endpgm 438define amdgpu_kernel void @merge_global_store_4_adjacent_loads_i8_natural_align(i8 addrspace(1)* %out, i8 addrspace(1)* %in) #0 { 439 %out.gep.1 = getelementptr i8, i8 addrspace(1)* %out, i8 1 440 %out.gep.2 = getelementptr i8, i8 addrspace(1)* %out, i8 2 441 %out.gep.3 = getelementptr i8, i8 addrspace(1)* %out, i8 3 442 %in.gep.1 = getelementptr i8, i8 addrspace(1)* %in, i8 1 443 %in.gep.2 = getelementptr i8, i8 addrspace(1)* %in, i8 2 444 %in.gep.3 = getelementptr i8, i8 addrspace(1)* %in, i8 3 445 446 %x = load i8, i8 addrspace(1)* %in 447 %y = load i8, i8 addrspace(1)* %in.gep.1 448 %z = load i8, i8 addrspace(1)* %in.gep.2 449 %w = load i8, i8 addrspace(1)* %in.gep.3 450 451 store i8 %x, i8 addrspace(1)* %out 452 store i8 %y, i8 addrspace(1)* %out.gep.1 453 store i8 %z, i8 addrspace(1)* %out.gep.2 454 store i8 %w, i8 addrspace(1)* %out.gep.3 455 ret void 456} 457 458; GCN-LABEL: {{^}}merge_global_store_4_vector_elts_loads_v4i32: 459; GCN: buffer_load_dwordx4 [[LOAD:v\[[0-9]+:[0-9]+\]]] 460; GCN: buffer_store_dwordx4 [[LOAD]] 461; GCN: s_endpgm 462define amdgpu_kernel void @merge_global_store_4_vector_elts_loads_v4i32(i32 addrspace(1)* %out, <4 x i32> addrspace(1)* %in) #0 { 463 %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1 464 %out.gep.2 = getelementptr i32, i32 addrspace(1)* %out, i32 2 465 %out.gep.3 = getelementptr i32, i32 addrspace(1)* %out, i32 3 466 %vec = load <4 x i32>, <4 x i32> addrspace(1)* %in 467 468 %x = extractelement <4 x i32> %vec, i32 0 469 %y = extractelement <4 x i32> %vec, i32 1 470 %z = extractelement <4 x i32> %vec, i32 2 471 %w = extractelement <4 x i32> %vec, i32 3 472 473 store i32 %x, i32 addrspace(1)* %out 474 store i32 %y, i32 addrspace(1)* %out.gep.1 475 store i32 %z, i32 addrspace(1)* %out.gep.2 476 store i32 %w, i32 addrspace(1)* %out.gep.3 477 ret void 478} 479 480; GCN-LABEL: {{^}}merge_local_store_2_constants_i8: 481; GCN: ds_write_b16 482; GCN: s_endpgm 483define amdgpu_kernel void @merge_local_store_2_constants_i8(i8 addrspace(3)* %out) #0 { 484 %out.gep.1 = getelementptr i8, i8 addrspace(3)* %out, i32 1 485 486 store i8 123, i8 addrspace(3)* %out.gep.1 487 store i8 456, i8 addrspace(3)* %out, align 2 488 ret void 489} 490 491; GCN-LABEL: {{^}}merge_local_store_2_constants_i32: 492; GCN-DAG: v_mov_b32_e32 v[[LO:[0-9]+]], 0x1c8 493; GCN-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], 0x7b 494; GCN: ds_write2_b32 v{{[0-9]+}}, v[[LO]], v[[HI]] offset1:1{{$}} 495define amdgpu_kernel void @merge_local_store_2_constants_i32(i32 addrspace(3)* %out) #0 { 496 %out.gep.1 = getelementptr i32, i32 addrspace(3)* %out, i32 1 497 498 store i32 123, i32 addrspace(3)* %out.gep.1 499 store i32 456, i32 addrspace(3)* %out 500 ret void 501} 502 503; GCN-LABEL: {{^}}merge_local_store_4_constants_i32: 504; GCN-DAG: v_mov_b32_e32 [[K2:v[0-9]+]], 0x1c8 505; GCN-DAG: v_mov_b32_e32 [[K3:v[0-9]+]], 0x14d 506; GCN-DAG: ds_write2_b32 v{{[0-9]+}}, [[K2]], [[K3]] offset0:2 offset1:3 507 508; GCN-DAG: v_mov_b32_e32 [[K0:v[0-9]+]], 0x4d2 509; GCN-DAG: v_mov_b32_e32 [[K1:v[0-9]+]], 0x7b 510; GCN-DAG: ds_write2_b32 v{{[0-9]+}}, [[K0]], [[K1]] offset1:1 511 512; GCN: s_endpgm 513define amdgpu_kernel void @merge_local_store_4_constants_i32(i32 addrspace(3)* %out) #0 { 514 %out.gep.1 = getelementptr i32, i32 addrspace(3)* %out, i32 1 515 %out.gep.2 = getelementptr i32, i32 addrspace(3)* %out, i32 2 516 %out.gep.3 = getelementptr i32, i32 addrspace(3)* %out, i32 3 517 518 store i32 123, i32 addrspace(3)* %out.gep.1 519 store i32 456, i32 addrspace(3)* %out.gep.2 520 store i32 333, i32 addrspace(3)* %out.gep.3 521 store i32 1234, i32 addrspace(3)* %out 522 ret void 523} 524 525; GCN-LABEL: {{^}}merge_global_store_5_constants_i32: 526; GCN-DAG: v_mov_b32_e32 v[[LO:[0-9]+]], 9{{$}} 527; GCN-DAG: v_mov_b32_e32 v[[HI4:[0-9]+]], -12{{$}} 528; GCN: buffer_store_dwordx4 v{{\[}}[[LO]]:[[HI4]]{{\]}} 529; GCN: v_mov_b32_e32 v[[HI:[0-9]+]], 11{{$}} 530; GCN: buffer_store_dword v[[HI]] 531define amdgpu_kernel void @merge_global_store_5_constants_i32(i32 addrspace(1)* %out) { 532 store i32 9, i32 addrspace(1)* %out, align 4 533 %idx1 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 1 534 store i32 12, i32 addrspace(1)* %idx1, align 4 535 %idx2 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 2 536 store i32 16, i32 addrspace(1)* %idx2, align 4 537 %idx3 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 3 538 store i32 -12, i32 addrspace(1)* %idx3, align 4 539 %idx4 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 4 540 store i32 11, i32 addrspace(1)* %idx4, align 4 541 ret void 542} 543 544; GCN-LABEL: {{^}}merge_global_store_6_constants_i32: 545; GCN: buffer_store_dwordx4 546; GCN: buffer_store_dwordx2 547define amdgpu_kernel void @merge_global_store_6_constants_i32(i32 addrspace(1)* %out) { 548 store i32 13, i32 addrspace(1)* %out, align 4 549 %idx1 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 1 550 store i32 15, i32 addrspace(1)* %idx1, align 4 551 %idx2 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 2 552 store i32 62, i32 addrspace(1)* %idx2, align 4 553 %idx3 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 3 554 store i32 63, i32 addrspace(1)* %idx3, align 4 555 %idx4 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 4 556 store i32 11, i32 addrspace(1)* %idx4, align 4 557 %idx5 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 5 558 store i32 123, i32 addrspace(1)* %idx5, align 4 559 ret void 560} 561 562; GCN-LABEL: {{^}}merge_global_store_7_constants_i32: 563; GCN: buffer_store_dwordx4 564; GCN: buffer_store_dwordx3 565define amdgpu_kernel void @merge_global_store_7_constants_i32(i32 addrspace(1)* %out) { 566 store i32 34, i32 addrspace(1)* %out, align 4 567 %idx1 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 1 568 store i32 999, i32 addrspace(1)* %idx1, align 4 569 %idx2 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 2 570 store i32 65, i32 addrspace(1)* %idx2, align 4 571 %idx3 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 3 572 store i32 33, i32 addrspace(1)* %idx3, align 4 573 %idx4 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 4 574 store i32 98, i32 addrspace(1)* %idx4, align 4 575 %idx5 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 5 576 store i32 91, i32 addrspace(1)* %idx5, align 4 577 %idx6 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 6 578 store i32 212, i32 addrspace(1)* %idx6, align 4 579 ret void 580} 581 582; GCN-LABEL: {{^}}merge_global_store_8_constants_i32: 583; GCN: buffer_store_dwordx4 584; GCN: buffer_store_dwordx4 585; GCN: s_endpgm 586define amdgpu_kernel void @merge_global_store_8_constants_i32(i32 addrspace(1)* %out) { 587 store i32 34, i32 addrspace(1)* %out, align 4 588 %idx1 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 1 589 store i32 999, i32 addrspace(1)* %idx1, align 4 590 %idx2 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 2 591 store i32 65, i32 addrspace(1)* %idx2, align 4 592 %idx3 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 3 593 store i32 33, i32 addrspace(1)* %idx3, align 4 594 %idx4 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 4 595 store i32 98, i32 addrspace(1)* %idx4, align 4 596 %idx5 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 5 597 store i32 91, i32 addrspace(1)* %idx5, align 4 598 %idx6 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 6 599 store i32 212, i32 addrspace(1)* %idx6, align 4 600 %idx7 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 7 601 store i32 999, i32 addrspace(1)* %idx7, align 4 602 ret void 603} 604 605; This requires handling of scalar_to_vector for v2i64 to avoid 606; scratch usage. 607; FIXME: Should do single load and store 608 609; GCN-LABEL: {{^}}copy_v3i32_align4: 610; GCN-NOT: SCRATCH_RSRC_DWORD 611; GCN-DAG: buffer_load_dwordx3 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}} 612; GCN-NOT: offen 613; GCN: s_waitcnt vmcnt 614; GCN-NOT: offen 615; GCN-DAG: buffer_store_dwordx3 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}} 616 617; GCN: ScratchSize: 0{{$}} 618define amdgpu_kernel void @copy_v3i32_align4(<3 x i32> addrspace(1)* noalias %out, <3 x i32> addrspace(1)* noalias %in) #0 { 619 %vec = load <3 x i32>, <3 x i32> addrspace(1)* %in, align 4 620 store <3 x i32> %vec, <3 x i32> addrspace(1)* %out 621 ret void 622} 623 624; GCN-LABEL: {{^}}copy_v3i64_align4: 625; GCN-NOT: SCRATCH_RSRC_DWORD 626; GCN-DAG: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}} 627; GCN-DAG: buffer_load_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:16{{$}} 628; GCN-NOT: offen 629; GCN: s_waitcnt vmcnt 630; GCN-NOT: offen 631; GCN-DAG: buffer_store_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}} 632; GCN-DAG: buffer_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:16{{$}} 633; GCN: ScratchSize: 0{{$}} 634define amdgpu_kernel void @copy_v3i64_align4(<3 x i64> addrspace(1)* noalias %out, <3 x i64> addrspace(1)* noalias %in) #0 { 635 %vec = load <3 x i64>, <3 x i64> addrspace(1)* %in, align 4 636 store <3 x i64> %vec, <3 x i64> addrspace(1)* %out 637 ret void 638} 639 640; GCN-LABEL: {{^}}copy_v3f32_align4: 641; GCN-NOT: SCRATCH_RSRC_DWORD 642; GCN-DAG: buffer_load_dwordx3 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}} 643; GCN-NOT: offen 644; GCN: s_waitcnt vmcnt 645; GCN-NOT: offen 646; GCN-DAG: buffer_store_dwordx3 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}} 647; GCN: ScratchSize: 0{{$}} 648define amdgpu_kernel void @copy_v3f32_align4(<3 x float> addrspace(1)* noalias %out, <3 x float> addrspace(1)* noalias %in) #0 { 649 %vec = load <3 x float>, <3 x float> addrspace(1)* %in, align 4 650 %fadd = fadd <3 x float> %vec, <float 1.0, float 2.0, float 4.0> 651 store <3 x float> %fadd, <3 x float> addrspace(1)* %out 652 ret void 653} 654 655; GCN-LABEL: {{^}}copy_v3f64_align4: 656; GCN-NOT: SCRATCH_RSRC_DWORD 657; GCN-DAG: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}} 658; GCN-DAG: buffer_load_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:16{{$}} 659; GCN-NOT: offen 660; GCN: s_waitcnt vmcnt 661; GCN-NOT: offen 662; GCN-DAG: buffer_store_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}} 663; GCN-DAG: buffer_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:16{{$}} 664; GCN: ScratchSize: 0{{$}} 665define amdgpu_kernel void @copy_v3f64_align4(<3 x double> addrspace(1)* noalias %out, <3 x double> addrspace(1)* noalias %in) #0 { 666 %vec = load <3 x double>, <3 x double> addrspace(1)* %in, align 4 667 %fadd = fadd <3 x double> %vec, <double 1.0, double 2.0, double 4.0> 668 store <3 x double> %fadd, <3 x double> addrspace(1)* %out 669 ret void 670} 671 672declare void @llvm.amdgcn.s.barrier() #1 673 674attributes #0 = { nounwind } 675attributes #1 = { convergent nounwind } 676