1; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=tahiti < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-SI -check-prefix=OPT-SICIVI %s 2; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=bonaire < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-CI -check-prefix=OPT-SICIVI %s 3; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-VI -check-prefix=OPT-SICIVI %s 4; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=gfx900 < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-GFX9 %s 5; RUN: llc -march=amdgcn -mcpu=tahiti -mattr=-promote-alloca -amdgpu-scalarize-global-loads=false -amdgpu-sroa=0 < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=SICIVI %s 6; RUN: llc -march=amdgcn -mcpu=bonaire -mattr=-promote-alloca -amdgpu-scalarize-global-loads=false -amdgpu-sroa=0 < %s | FileCheck -check-prefix=GCN -check-prefix=CI -check-prefix=SICIVI %s 7; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -amdgpu-scalarize-global-loads=false -mattr=-promote-alloca -amdgpu-sroa=0 < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=SICIVI %s 8; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-promote-alloca -amdgpu-scalarize-global-loads=false -amdgpu-sroa=0 < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s 9 10target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5" 11 12; OPT-LABEL: @test_sink_global_small_offset_i32( 13; OPT-CI-NOT: getelementptr i32, i32 addrspace(1)* %in 14; OPT-VI: getelementptr i32, i32 addrspace(1)* %in 15; OPT: br i1 16; OPT-CI: getelementptr i8, 17 18; GCN-LABEL: {{^}}test_sink_global_small_offset_i32: 19define amdgpu_kernel void @test_sink_global_small_offset_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) { 20entry: 21 %out.gep = getelementptr i32, i32 addrspace(1)* %out, i64 999999 22 %in.gep = getelementptr i32, i32 addrspace(1)* %in, i64 7 23 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 24 %tmp0 = icmp eq i32 %tid, 0 25 br i1 %tmp0, label %endif, label %if 26 27if: 28 %tmp1 = load i32, i32 addrspace(1)* %in.gep 29 br label %endif 30 31endif: 32 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ] 33 store i32 %x, i32 addrspace(1)* %out.gep 34 br label %done 35 36done: 37 ret void 38} 39 40; OPT-LABEL: @test_sink_global_small_max_i32_ds_offset( 41; OPT: %in.gep = getelementptr i8, i8 addrspace(1)* %in, i64 65535 42; OPT: br i1 43 44; GCN-LABEL: {{^}}test_sink_global_small_max_i32_ds_offset: 45; GCN: s_and_saveexec_b64 46; SICIVI: buffer_load_sbyte {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, s{{[0-9]+$}} 47 48; GFX9: v_add_co_u32_e32 v{{[0-9]+}}, vcc, 0xf000, 49; GFX9: v_addc_co_u32_e32 v{{[0-9]+}}, vcc, 0, 50; GFX9: global_load_sbyte {{v[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, off offset:4095{{$}} 51; GCN: {{^}}BB1_2: 52; GCN: s_or_b64 exec 53define amdgpu_kernel void @test_sink_global_small_max_i32_ds_offset(i32 addrspace(1)* %out, i8 addrspace(1)* %in) { 54entry: 55 %out.gep = getelementptr i32, i32 addrspace(1)* %out, i64 99999 56 %in.gep = getelementptr i8, i8 addrspace(1)* %in, i64 65535 57 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 58 %tmp0 = icmp eq i32 %tid, 0 59 br i1 %tmp0, label %endif, label %if 60 61if: 62 %tmp1 = load i8, i8 addrspace(1)* %in.gep 63 %tmp2 = sext i8 %tmp1 to i32 64 br label %endif 65 66endif: 67 %x = phi i32 [ %tmp2, %if ], [ 0, %entry ] 68 store i32 %x, i32 addrspace(1)* %out.gep 69 br label %done 70 71done: 72 ret void 73} 74 75; GCN-LABEL: {{^}}test_sink_global_small_max_mubuf_offset: 76; GCN: s_and_saveexec_b64 77; SICIVI: buffer_load_sbyte {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:4095{{$}} 78; GFX9: global_load_sbyte {{v[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, off offset:4095{{$}} 79; GCN: {{^}}BB2_2: 80; GCN: s_or_b64 exec 81define amdgpu_kernel void @test_sink_global_small_max_mubuf_offset(i32 addrspace(1)* %out, i8 addrspace(1)* %in) { 82entry: 83 %out.gep = getelementptr i32, i32 addrspace(1)* %out, i32 1024 84 %in.gep = getelementptr i8, i8 addrspace(1)* %in, i64 4095 85 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 86 %tmp0 = icmp eq i32 %tid, 0 87 br i1 %tmp0, label %endif, label %if 88 89if: 90 %tmp1 = load i8, i8 addrspace(1)* %in.gep 91 %tmp2 = sext i8 %tmp1 to i32 92 br label %endif 93 94endif: 95 %x = phi i32 [ %tmp2, %if ], [ 0, %entry ] 96 store i32 %x, i32 addrspace(1)* %out.gep 97 br label %done 98 99done: 100 ret void 101} 102 103; GCN-LABEL: {{^}}test_sink_global_small_max_plus_1_mubuf_offset: 104; GCN: s_and_saveexec_b64 105; SICIVI: buffer_load_sbyte {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, s{{[0-9]+$}} 106; GFX9: global_load_sbyte {{v[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, off{{$}} 107; GCN: {{^}}BB3_2: 108; GCN: s_or_b64 exec 109define amdgpu_kernel void @test_sink_global_small_max_plus_1_mubuf_offset(i32 addrspace(1)* %out, i8 addrspace(1)* %in) { 110entry: 111 %out.gep = getelementptr i32, i32 addrspace(1)* %out, i64 99999 112 %in.gep = getelementptr i8, i8 addrspace(1)* %in, i64 4096 113 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 114 %tmp0 = icmp eq i32 %tid, 0 115 br i1 %tmp0, label %endif, label %if 116 117if: 118 %tmp1 = load i8, i8 addrspace(1)* %in.gep 119 %tmp2 = sext i8 %tmp1 to i32 120 br label %endif 121 122endif: 123 %x = phi i32 [ %tmp2, %if ], [ 0, %entry ] 124 store i32 %x, i32 addrspace(1)* %out.gep 125 br label %done 126 127done: 128 ret void 129} 130 131; OPT-LABEL: @test_sink_scratch_small_offset_i32( 132; OPT-NOT: getelementptr [512 x i32] 133; OPT: br i1 134; OPT: getelementptr i8, 135 136; GCN-LABEL: {{^}}test_sink_scratch_small_offset_i32: 137; GCN: s_and_saveexec_b64 138; GCN: buffer_store_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offset:4092{{$}} 139; GCN: buffer_load_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offset:4092{{$}} 140; GCN: {{^}}BB4_2: 141define amdgpu_kernel void @test_sink_scratch_small_offset_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in, i32 %arg) { 142entry: 143 %alloca = alloca [512 x i32], align 4, addrspace(5) 144 %out.gep.0 = getelementptr i32, i32 addrspace(1)* %out, i64 999998 145 %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i64 999999 146 %add.arg = add i32 %arg, 8 147 %alloca.gep = getelementptr [512 x i32], [512 x i32] addrspace(5)* %alloca, i32 0, i32 1022 148 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 149 %tmp0 = icmp eq i32 %tid, 0 150 br i1 %tmp0, label %endif, label %if 151 152if: 153 store volatile i32 123, i32 addrspace(5)* %alloca.gep 154 %tmp1 = load volatile i32, i32 addrspace(5)* %alloca.gep 155 br label %endif 156 157endif: 158 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ] 159 store i32 %x, i32 addrspace(1)* %out.gep.0 160 %load = load volatile i32, i32 addrspace(5)* %alloca.gep 161 store i32 %load, i32 addrspace(1)* %out.gep.1 162 br label %done 163 164done: 165 ret void 166} 167 168; This ends up not fitting due to the reserved 4 bytes at offset 0 169; OPT-LABEL: @test_sink_scratch_small_offset_i32_reserved( 170; OPT-NOT: getelementptr [512 x i32] 171; OPT: br i1 172; OPT: getelementptr i8, 173 174; GCN-LABEL: {{^}}test_sink_scratch_small_offset_i32_reserved: 175; GCN: s_and_saveexec_b64 176; GCN: v_mov_b32_e32 [[BASE_FI0:v[0-9]+]], 4 177; GCN: buffer_store_dword {{v[0-9]+}}, [[BASE_FI0]], {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen offset:4092{{$}} 178; GCN: v_mov_b32_e32 [[BASE_FI1:v[0-9]+]], 4 179; GCN: buffer_load_dword {{v[0-9]+}}, [[BASE_FI1]], {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen offset:4092{{$}} 180; GCN: {{^BB[0-9]+}}_2: 181 182define amdgpu_kernel void @test_sink_scratch_small_offset_i32_reserved(i32 addrspace(1)* %out, i32 addrspace(1)* %in, i32 %arg) { 183entry: 184 %alloca = alloca [512 x i32], align 4, addrspace(5) 185 %out.gep.0 = getelementptr i32, i32 addrspace(1)* %out, i64 999998 186 %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i64 999999 187 %add.arg = add i32 %arg, 8 188 %alloca.gep = getelementptr [512 x i32], [512 x i32] addrspace(5)* %alloca, i32 0, i32 1023 189 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 190 %tmp0 = icmp eq i32 %tid, 0 191 br i1 %tmp0, label %endif, label %if 192 193if: 194 store volatile i32 123, i32 addrspace(5)* %alloca.gep 195 %tmp1 = load volatile i32, i32 addrspace(5)* %alloca.gep 196 br label %endif 197 198endif: 199 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ] 200 store i32 %x, i32 addrspace(1)* %out.gep.0 201 %load = load volatile i32, i32 addrspace(5)* %alloca.gep 202 store i32 %load, i32 addrspace(1)* %out.gep.1 203 br label %done 204 205done: 206 ret void 207} 208 209; OPT-LABEL: @test_no_sink_scratch_large_offset_i32( 210; OPT: %alloca.gep = getelementptr [512 x i32], [512 x i32] addrspace(5)* %alloca, i32 0, i32 1024 211; OPT: br i1 212; OPT-NOT: ptrtoint 213 214; GCN-LABEL: {{^}}test_no_sink_scratch_large_offset_i32: 215; GCN: s_and_saveexec_b64 216; GCN: buffer_store_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen{{$}} 217; GCN: buffer_load_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen{{$}} 218; GCN: {{^BB[0-9]+}}_2: 219define amdgpu_kernel void @test_no_sink_scratch_large_offset_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in, i32 %arg) { 220entry: 221 %alloca = alloca [512 x i32], align 4, addrspace(5) 222 %out.gep.0 = getelementptr i32, i32 addrspace(1)* %out, i64 999998 223 %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i64 999999 224 %add.arg = add i32 %arg, 8 225 %alloca.gep = getelementptr [512 x i32], [512 x i32] addrspace(5)* %alloca, i32 0, i32 1024 226 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 227 %tmp0 = icmp eq i32 %tid, 0 228 br i1 %tmp0, label %endif, label %if 229 230if: 231 store volatile i32 123, i32 addrspace(5)* %alloca.gep 232 %tmp1 = load volatile i32, i32 addrspace(5)* %alloca.gep 233 br label %endif 234 235endif: 236 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ] 237 store i32 %x, i32 addrspace(1)* %out.gep.0 238 %load = load volatile i32, i32 addrspace(5)* %alloca.gep 239 store i32 %load, i32 addrspace(1)* %out.gep.1 240 br label %done 241 242done: 243 ret void 244} 245 246; GCN-LABEL: {{^}}test_sink_global_vreg_sreg_i32: 247; GCN: s_and_saveexec_b64 248; CI: buffer_load_dword {{v[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}} 249; VI: flat_load_dword v{{[0-9]+}}, v[{{[0-9]+:[0-9]+}}] 250; GCN: {{^BB[0-9]+}}_2: 251define amdgpu_kernel void @test_sink_global_vreg_sreg_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in, i32 %offset) { 252entry: 253 %offset.ext = zext i32 %offset to i64 254 %out.gep = getelementptr i32, i32 addrspace(1)* %out, i64 999999 255 %in.gep = getelementptr i32, i32 addrspace(1)* %in, i64 %offset.ext 256 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 257 %tmp0 = icmp eq i32 %tid, 0 258 br i1 %tmp0, label %endif, label %if 259 260if: 261 %tmp1 = load i32, i32 addrspace(1)* %in.gep 262 br label %endif 263 264endif: 265 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ] 266 store i32 %x, i32 addrspace(1)* %out.gep 267 br label %done 268 269done: 270 ret void 271} 272 273; OPT-LABEL: @test_sink_constant_small_offset_i32 274; OPT-NOT: getelementptr i32, i32 addrspace(4)* 275; OPT: br i1 276 277; GCN-LABEL: {{^}}test_sink_constant_small_offset_i32: 278; GCN: s_and_saveexec_b64 279; SI: s_load_dword s{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0x7{{$}} 280; GCN: s_or_b64 exec, exec 281define amdgpu_kernel void @test_sink_constant_small_offset_i32(i32 addrspace(1)* %out, i32 addrspace(4)* %in) { 282entry: 283 %out.gep = getelementptr i32, i32 addrspace(1)* %out, i64 999999 284 %in.gep = getelementptr i32, i32 addrspace(4)* %in, i64 7 285 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 286 %tmp0 = icmp eq i32 %tid, 0 287 br i1 %tmp0, label %endif, label %if 288 289if: 290 %tmp1 = load i32, i32 addrspace(4)* %in.gep 291 br label %endif 292 293endif: 294 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ] 295 store i32 %x, i32 addrspace(1)* %out.gep 296 br label %done 297 298done: 299 ret void 300} 301 302; OPT-LABEL: @test_sink_constant_max_8_bit_offset_i32 303; OPT-NOT: getelementptr i32, i32 addrspace(4)* 304; OPT: br i1 305 306; GCN-LABEL: {{^}}test_sink_constant_max_8_bit_offset_i32: 307; GCN: s_and_saveexec_b64 308; SI: s_load_dword s{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0xff{{$}} 309; GCN: s_or_b64 exec, exec 310define amdgpu_kernel void @test_sink_constant_max_8_bit_offset_i32(i32 addrspace(1)* %out, i32 addrspace(4)* %in) { 311entry: 312 %out.gep = getelementptr i32, i32 addrspace(1)* %out, i64 999999 313 %in.gep = getelementptr i32, i32 addrspace(4)* %in, i64 255 314 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 315 %tmp0 = icmp eq i32 %tid, 0 316 br i1 %tmp0, label %endif, label %if 317 318if: 319 %tmp1 = load i32, i32 addrspace(4)* %in.gep 320 br label %endif 321 322endif: 323 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ] 324 store i32 %x, i32 addrspace(1)* %out.gep 325 br label %done 326 327done: 328 ret void 329} 330 331; OPT-LABEL: @test_sink_constant_max_8_bit_offset_p1_i32 332; OPT-SI: getelementptr i32, i32 addrspace(4)* 333; OPT-CI-NOT: getelementptr i32, i32 addrspace(4)* 334; OPT-VI-NOT: getelementptr i32, i32 addrspace(4)* 335; OPT: br i1 336 337; GCN-LABEL: {{^}}test_sink_constant_max_8_bit_offset_p1_i32: 338; GCN: s_and_saveexec_b64 339; SI: s_movk_i32 [[OFFSET:s[0-9]+]], 0x400 340 341; SI: s_load_dword s{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, [[OFFSET]]{{$}} 342; GCN: s_or_b64 exec, exec 343define amdgpu_kernel void @test_sink_constant_max_8_bit_offset_p1_i32(i32 addrspace(1)* %out, i32 addrspace(4)* %in) { 344entry: 345 %out.gep = getelementptr i32, i32 addrspace(1)* %out, i64 999999 346 %in.gep = getelementptr i32, i32 addrspace(4)* %in, i64 256 347 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 348 %tmp0 = icmp eq i32 %tid, 0 349 br i1 %tmp0, label %endif, label %if 350 351if: 352 %tmp1 = load i32, i32 addrspace(4)* %in.gep 353 br label %endif 354 355endif: 356 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ] 357 store i32 %x, i32 addrspace(1)* %out.gep 358 br label %done 359 360done: 361 ret void 362} 363 364; OPT-LABEL: @test_sink_constant_max_32_bit_offset_i32 365; OPT-SI: getelementptr i32, i32 addrspace(4)* 366; OPT-CI-NOT: getelementptr i32, i32 addrspace(4)* 367; OPT: br i1 368 369; GCN-LABEL: {{^}}test_sink_constant_max_32_bit_offset_i32: 370; GCN: s_and_saveexec_b64 371; SI: s_add_u32 s{{[0-9]+}}, s{{[0-9]+}}, -4{{$}} 372; SI: s_addc_u32 s{{[0-9]+}}, s{{[0-9]+}}, 3{{$}} 373; SI: s_load_dword s{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0x0{{$}} 374 375; VI: s_add_u32 s{{[0-9]+}}, s{{[0-9]+}}, -4{{$}} 376; VI: s_addc_u32 s{{[0-9]+}}, s{{[0-9]+}}, 3{{$}} 377; VI: s_load_dword s{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0x0{{$}} 378 379; CI: s_load_dword s{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0xffffffff{{$}} 380 381; GCN: s_or_b64 exec, exec 382define amdgpu_kernel void @test_sink_constant_max_32_bit_offset_i32(i32 addrspace(1)* %out, i32 addrspace(4)* %in) { 383entry: 384 %out.gep = getelementptr i32, i32 addrspace(1)* %out, i64 999999 385 %in.gep = getelementptr i32, i32 addrspace(4)* %in, i64 4294967295 386 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 387 %tmp0 = icmp eq i32 %tid, 0 388 br i1 %tmp0, label %endif, label %if 389 390if: 391 %tmp1 = load i32, i32 addrspace(4)* %in.gep 392 br label %endif 393 394endif: 395 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ] 396 store i32 %x, i32 addrspace(1)* %out.gep 397 br label %done 398 399done: 400 ret void 401} 402 403; OPT-LABEL: @test_sink_constant_max_32_bit_offset_p1_i32 404; OPT: getelementptr i32, i32 addrspace(4)* 405; OPT: br i1 406 407; GCN-LABEL: {{^}}test_sink_constant_max_32_bit_offset_p1_i32: 408; GCN: s_and_saveexec_b64 409; GCN: s_add_u32 410; GCN: s_addc_u32 411; SI: s_load_dword s{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0x0{{$}} 412; GCN: s_or_b64 exec, exec 413define amdgpu_kernel void @test_sink_constant_max_32_bit_offset_p1_i32(i32 addrspace(1)* %out, i32 addrspace(4)* %in) { 414entry: 415 %out.gep = getelementptr i32, i32 addrspace(1)* %out, i64 999999 416 %in.gep = getelementptr i32, i32 addrspace(4)* %in, i64 17179869181 417 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 418 %tmp0 = icmp eq i32 %tid, 0 419 br i1 %tmp0, label %endif, label %if 420 421if: 422 %tmp1 = load i32, i32 addrspace(4)* %in.gep 423 br label %endif 424 425endif: 426 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ] 427 store i32 %x, i32 addrspace(1)* %out.gep 428 br label %done 429 430done: 431 ret void 432} 433 434; GCN-LABEL: {{^}}test_sink_constant_max_20_bit_byte_offset_i32: 435; GCN: s_and_saveexec_b64 436; SI: s_mov_b32 [[OFFSET:s[0-9]+]], 0xffffc{{$}} 437; SI: s_load_dword s{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, [[OFFSET]]{{$}} 438 439; CI: s_load_dword s{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0x3ffff{{$}} 440; VI: s_load_dword s{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0xffffc{{$}} 441 442; GCN: s_or_b64 exec, exec 443define amdgpu_kernel void @test_sink_constant_max_20_bit_byte_offset_i32(i32 addrspace(1)* %out, i32 addrspace(4)* %in) { 444entry: 445 %out.gep = getelementptr i32, i32 addrspace(1)* %out, i64 999999 446 %in.gep = getelementptr i32, i32 addrspace(4)* %in, i64 262143 447 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 448 %tmp0 = icmp eq i32 %tid, 0 449 br i1 %tmp0, label %endif, label %if 450 451if: 452 %tmp1 = load i32, i32 addrspace(4)* %in.gep 453 br label %endif 454 455endif: 456 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ] 457 store i32 %x, i32 addrspace(1)* %out.gep 458 br label %done 459 460done: 461 ret void 462} 463 464; OPT-LABEL: @test_sink_constant_max_20_bit_byte_offset_p1_i32 465; OPT-SI: getelementptr i32, i32 addrspace(4)* 466; OPT-CI-NOT: getelementptr i32, i32 addrspace(4)* 467; OPT-VI: getelementptr i32, i32 addrspace(4)* 468; OPT: br i1 469 470; GCN-LABEL: {{^}}test_sink_constant_max_20_bit_byte_offset_p1_i32: 471; GCN: s_and_saveexec_b64 472; SI: s_mov_b32 [[OFFSET:s[0-9]+]], 0x100000{{$}} 473; SI: s_load_dword s{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, [[OFFSET]]{{$}} 474 475; CI: s_load_dword s{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0x40000{{$}} 476 477; VI: s_mov_b32 [[OFFSET:s[0-9]+]], 0x100000{{$}} 478; VI: s_load_dword s{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, [[OFFSET]]{{$}} 479 480; GCN: s_or_b64 exec, exec 481define amdgpu_kernel void @test_sink_constant_max_20_bit_byte_offset_p1_i32(i32 addrspace(1)* %out, i32 addrspace(4)* %in) { 482entry: 483 %out.gep = getelementptr i32, i32 addrspace(1)* %out, i64 999999 484 %in.gep = getelementptr i32, i32 addrspace(4)* %in, i64 262144 485 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 486 %tmp0 = icmp eq i32 %tid, 0 487 br i1 %tmp0, label %endif, label %if 488 489if: 490 %tmp1 = load i32, i32 addrspace(4)* %in.gep 491 br label %endif 492 493endif: 494 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ] 495 store i32 %x, i32 addrspace(1)* %out.gep 496 br label %done 497 498done: 499 ret void 500} 501 502%struct.foo = type { [3 x float], [3 x float] } 503 504; OPT-LABEL: @sink_ds_address( 505; OPT: getelementptr inbounds i8, 506 507; GCN-LABEL: {{^}}sink_ds_address: 508; GCN: s_load_dword [[SREG1:s[0-9]+]], 509; GCN: v_mov_b32_e32 [[VREG1:v[0-9]+]], [[SREG1]] 510; GCN-DAG: ds_read2_b32 v[{{[0-9+:[0-9]+}}], [[VREG1]] offset0:3 offset1:5 511define amdgpu_kernel void @sink_ds_address(%struct.foo addrspace(3)* nocapture %ptr) nounwind { 512entry: 513 %x = getelementptr inbounds %struct.foo, %struct.foo addrspace(3)* %ptr, i32 0, i32 1, i32 0 514 %y = getelementptr inbounds %struct.foo, %struct.foo addrspace(3)* %ptr, i32 0, i32 1, i32 2 515 br label %bb32 516 517bb32: 518 %a = load float, float addrspace(3)* %x, align 4 519 %b = load float, float addrspace(3)* %y, align 4 520 %cmp = fcmp one float %a, %b 521 br i1 %cmp, label %bb34, label %bb33 522 523bb33: 524 unreachable 525 526bb34: 527 unreachable 528} 529 530; Address offset is not a multiple of 4. This is a valid mubuf offset, 531; but not smrd. 532 533; OPT-LABEL: @test_sink_constant_small_max_mubuf_offset_load_i32_align_1( 534; OPT: br i1 %tmp0, 535; OPT: if: 536; OPT: getelementptr i8, {{.*}} 4095 537define amdgpu_kernel void @test_sink_constant_small_max_mubuf_offset_load_i32_align_1(i32 addrspace(1)* %out, i8 addrspace(4)* %in) { 538entry: 539 %out.gep = getelementptr i32, i32 addrspace(1)* %out, i32 1024 540 %in.gep = getelementptr i8, i8 addrspace(4)* %in, i64 4095 541 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 542 %tmp0 = icmp eq i32 %tid, 0 543 br i1 %tmp0, label %endif, label %if 544 545if: 546 %bitcast = bitcast i8 addrspace(4)* %in.gep to i32 addrspace(4)* 547 %tmp1 = load i32, i32 addrspace(4)* %bitcast, align 1 548 br label %endif 549 550endif: 551 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ] 552 store i32 %x, i32 addrspace(1)* %out.gep 553 br label %done 554 555done: 556 ret void 557} 558 559; OPT-LABEL: @test_sink_local_small_offset_atomicrmw_i32( 560; OPT: %0 = bitcast i32 addrspace(3)* %in to i8 addrspace(3)* 561; OPT: %sunkaddr = getelementptr i8, i8 addrspace(3)* %0, i32 28 562; OPT: %1 = bitcast i8 addrspace(3)* %sunkaddr to i32 addrspace(3)* 563; OPT: %tmp1 = atomicrmw add i32 addrspace(3)* %1, i32 2 seq_cst 564define amdgpu_kernel void @test_sink_local_small_offset_atomicrmw_i32(i32 addrspace(3)* %out, i32 addrspace(3)* %in) { 565entry: 566 %out.gep = getelementptr i32, i32 addrspace(3)* %out, i32 999999 567 %in.gep = getelementptr i32, i32 addrspace(3)* %in, i32 7 568 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 569 %tmp0 = icmp eq i32 %tid, 0 570 br i1 %tmp0, label %endif, label %if 571 572if: 573 %tmp1 = atomicrmw add i32 addrspace(3)* %in.gep, i32 2 seq_cst 574 br label %endif 575 576endif: 577 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ] 578 store i32 %x, i32 addrspace(3)* %out.gep 579 br label %done 580 581done: 582 ret void 583} 584 585; OPT-LABEL: @test_sink_local_small_offset_cmpxchg_i32( 586; OPT: %0 = bitcast i32 addrspace(3)* %in to i8 addrspace(3)* 587; OPT: %sunkaddr = getelementptr i8, i8 addrspace(3)* %0, i32 28 588; OPT: %1 = bitcast i8 addrspace(3)* %sunkaddr to i32 addrspace(3)* 589; OPT: %tmp1.struct = cmpxchg i32 addrspace(3)* %1, i32 undef, i32 2 seq_cst monotonic 590define amdgpu_kernel void @test_sink_local_small_offset_cmpxchg_i32(i32 addrspace(3)* %out, i32 addrspace(3)* %in) { 591entry: 592 %out.gep = getelementptr i32, i32 addrspace(3)* %out, i32 999999 593 %in.gep = getelementptr i32, i32 addrspace(3)* %in, i32 7 594 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 595 %tmp0 = icmp eq i32 %tid, 0 596 br i1 %tmp0, label %endif, label %if 597 598if: 599 %tmp1.struct = cmpxchg i32 addrspace(3)* %in.gep, i32 undef, i32 2 seq_cst monotonic 600 %tmp1 = extractvalue { i32, i1 } %tmp1.struct, 0 601 br label %endif 602 603endif: 604 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ] 605 store i32 %x, i32 addrspace(3)* %out.gep 606 br label %done 607 608done: 609 ret void 610} 611 612; OPT-LABEL: @test_wrong_operand_local_small_offset_cmpxchg_i32( 613; OPT: %in.gep = getelementptr i32, i32 addrspace(3)* %in, i32 7 614; OPT: br i1 615; OPT: cmpxchg i32 addrspace(3)* addrspace(3)* undef, i32 addrspace(3)* %in.gep, i32 addrspace(3)* undef seq_cst monotonic 616define amdgpu_kernel void @test_wrong_operand_local_small_offset_cmpxchg_i32(i32 addrspace(3)* addrspace(3)* %out, i32 addrspace(3)* %in) { 617entry: 618 %out.gep = getelementptr i32 addrspace(3)*, i32 addrspace(3)* addrspace(3)* %out, i32 999999 619 %in.gep = getelementptr i32, i32 addrspace(3)* %in, i32 7 620 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 621 %tmp0 = icmp eq i32 %tid, 0 622 br i1 %tmp0, label %endif, label %if 623 624if: 625 %tmp1.struct = cmpxchg i32 addrspace(3)* addrspace(3)* undef, i32 addrspace(3)* %in.gep, i32 addrspace(3)* undef seq_cst monotonic 626 %tmp1 = extractvalue { i32 addrspace(3)*, i1 } %tmp1.struct, 0 627 br label %endif 628 629endif: 630 %x = phi i32 addrspace(3)* [ %tmp1, %if ], [ null, %entry ] 631 store i32 addrspace(3)* %x, i32 addrspace(3)* addrspace(3)* %out.gep 632 br label %done 633 634done: 635 ret void 636} 637 638; OPT-LABEL: @test_sink_local_small_offset_atomic_inc_i32( 639; OPT: %0 = bitcast i32 addrspace(3)* %in to i8 addrspace(3)* 640; OPT: %sunkaddr = getelementptr i8, i8 addrspace(3)* %0, i32 28 641; OPT: %1 = bitcast i8 addrspace(3)* %sunkaddr to i32 addrspace(3)* 642; OPT: %tmp1 = call i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* %1, i32 2, i32 0, i32 0, i1 false) 643define amdgpu_kernel void @test_sink_local_small_offset_atomic_inc_i32(i32 addrspace(3)* %out, i32 addrspace(3)* %in) { 644entry: 645 %out.gep = getelementptr i32, i32 addrspace(3)* %out, i32 999999 646 %in.gep = getelementptr i32, i32 addrspace(3)* %in, i32 7 647 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 648 %tmp0 = icmp eq i32 %tid, 0 649 br i1 %tmp0, label %endif, label %if 650 651if: 652 %tmp1 = call i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* %in.gep, i32 2, i32 0, i32 0, i1 false) 653 br label %endif 654 655endif: 656 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ] 657 store i32 %x, i32 addrspace(3)* %out.gep 658 br label %done 659 660done: 661 ret void 662} 663 664; OPT-LABEL: @test_sink_local_small_offset_atomic_dec_i32( 665; OPT: %0 = bitcast i32 addrspace(3)* %in to i8 addrspace(3)* 666; OPT: %sunkaddr = getelementptr i8, i8 addrspace(3)* %0, i32 28 667; OPT: %1 = bitcast i8 addrspace(3)* %sunkaddr to i32 addrspace(3)* 668; OPT: %tmp1 = call i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* %1, i32 2, i32 0, i32 0, i1 false) 669define amdgpu_kernel void @test_sink_local_small_offset_atomic_dec_i32(i32 addrspace(3)* %out, i32 addrspace(3)* %in) { 670entry: 671 %out.gep = getelementptr i32, i32 addrspace(3)* %out, i32 999999 672 %in.gep = getelementptr i32, i32 addrspace(3)* %in, i32 7 673 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 674 %tmp0 = icmp eq i32 %tid, 0 675 br i1 %tmp0, label %endif, label %if 676 677if: 678 %tmp1 = call i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* %in.gep, i32 2, i32 0, i32 0, i1 false) 679 br label %endif 680 681endif: 682 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ] 683 store i32 %x, i32 addrspace(3)* %out.gep 684 br label %done 685 686done: 687 ret void 688} 689 690; OPT-LABEL: @test_sink_global_small_min_scratch_global_offset( 691; OPT-SICIVI: %in.gep = getelementptr i8, i8 addrspace(1)* %in, i64 -4096 692; OPT-SICIV: br 693; OPT-SICIVI: %tmp1 = load i8, i8 addrspace(1)* %in.gep 694 695; OPT-GFX9: br 696; OPT-GFX9: %sunkaddr = getelementptr i8, i8 addrspace(1)* %in, i64 -4096 697; OPT-GFX9: load i8, i8 addrspace(1)* %sunkaddr 698 699; GCN-LABEL: {{^}}test_sink_global_small_min_scratch_global_offset: 700; GFX9: global_load_sbyte v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:-4096{{$}} 701define amdgpu_kernel void @test_sink_global_small_min_scratch_global_offset(i32 addrspace(1)* %out, i8 addrspace(1)* %in) { 702entry: 703 %out.gep = getelementptr i32, i32 addrspace(1)* %out, i32 1024 704 %in.gep = getelementptr i8, i8 addrspace(1)* %in, i64 -4096 705 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 706 %tmp0 = icmp eq i32 %tid, 0 707 br i1 %tmp0, label %endif, label %if 708 709if: 710 %tmp1 = load i8, i8 addrspace(1)* %in.gep 711 %tmp2 = sext i8 %tmp1 to i32 712 br label %endif 713 714endif: 715 %x = phi i32 [ %tmp2, %if ], [ 0, %entry ] 716 store i32 %x, i32 addrspace(1)* %out.gep 717 br label %done 718 719done: 720 ret void 721} 722 723; OPT-LABEL: @test_sink_global_small_min_scratch_global_neg1_offset( 724; OPT: %in.gep = getelementptr i8, i8 addrspace(1)* %in, i64 -4097 725; OPT: br 726; OPT: load i8, i8 addrspace(1)* %in.gep 727 728; GCN-LABEL: {{^}}test_sink_global_small_min_scratch_global_neg1_offset: 729define amdgpu_kernel void @test_sink_global_small_min_scratch_global_neg1_offset(i32 addrspace(1)* %out, i8 addrspace(1)* %in) { 730entry: 731 %out.gep = getelementptr i32, i32 addrspace(1)* %out, i64 99999 732 %in.gep = getelementptr i8, i8 addrspace(1)* %in, i64 -4097 733 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 734 %tmp0 = icmp eq i32 %tid, 0 735 br i1 %tmp0, label %endif, label %if 736 737if: 738 %tmp1 = load i8, i8 addrspace(1)* %in.gep 739 %tmp2 = sext i8 %tmp1 to i32 740 br label %endif 741 742endif: 743 %x = phi i32 [ %tmp2, %if ], [ 0, %entry ] 744 store i32 %x, i32 addrspace(1)* %out.gep 745 br label %done 746 747done: 748 ret void 749} 750 751declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #0 752declare i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* nocapture, i32, i32, i32, i1) #2 753declare i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* nocapture, i32, i32, i32, i1) #2 754 755attributes #0 = { nounwind readnone } 756attributes #1 = { nounwind } 757attributes #2 = { nounwind argmemonly } 758