1# RUN: llc -march=amdgcn -verify-machineinstrs -run-pass si-load-store-opt -o - %s | FileCheck %s
2
3# Check that SILoadStoreOptimizer honors memory dependencies between moved
4# instructions.
5#
6# The following IR snippet would usually be optimized by the peephole optimizer.
7# However, an equivalent situation can occur with buffer instructions as well.
8
9# CHECK-LABEL: name: mem_dependency
10# CHECK: DS_READ2_B32 %0, 0, 1,
11# CHECK: DS_WRITE_B32 %0, killed %1, 64,
12# CHECK: DS_READ2_B32 %0, 16, 17,
13# CHECK: DS_WRITE_B32 killed %0, %5, 0
14
15--- |
16  define amdgpu_kernel void @mem_dependency(i32 addrspace(3)* %ptr.0) nounwind {
17    %ptr.4 = getelementptr i32, i32 addrspace(3)* %ptr.0, i32 1
18    %ptr.64 = getelementptr i32, i32 addrspace(3)* %ptr.0, i32 16
19    %1 = load i32, i32 addrspace(3)* %ptr.0
20    store i32 %1, i32 addrspace(3)* %ptr.64
21    %2 = load i32, i32 addrspace(3)* %ptr.64
22    %3 = load i32, i32 addrspace(3)* %ptr.4
23    %4 = add i32 %2, %3
24    store i32 %4, i32 addrspace(3)* %ptr.0
25    ret void
26  }
27
28  @lds0 = external dso_local unnamed_addr addrspace(3) global [256 x i32], align 4
29  @lds1 = external dso_local unnamed_addr addrspace(3) global [256 x i32], align 4
30  @lds2 = external dso_local unnamed_addr addrspace(3) global [256 x i32], align 4
31  @lds3 = external dso_local unnamed_addr addrspace(3) global [256 x i32], align 4
32
33  define void @asm_defines_address() #0 {
34  bb:
35    %tmp1 = load i32, i32 addrspace(3)* getelementptr inbounds ([256 x i32], [256 x i32] addrspace(3)* @lds0, i32 0, i32 0), align 4
36    %0 = and i32 %tmp1, 255
37    %tmp3 = load i32, i32 addrspace(3)* getelementptr ([256 x i32], [256 x i32] addrspace(3)* @lds1, i32 0, i32 undef), align 4
38    %tmp6 = load i32, i32 addrspace(3)* getelementptr ([256 x i32], [256 x i32] addrspace(3)* @lds3, i32 0, i32 undef), align 4
39    %tmp7 = tail call i32 asm "v_or_b32 $0, 0, $1", "=v,v"(i32 %tmp6) #1
40    %tmp10 = lshr i32 %tmp7, 16
41    %tmp11 = and i32 %tmp10, 255
42    %tmp12 = getelementptr inbounds [256 x i32], [256 x i32] addrspace(3)* @lds1, i32 0, i32 %tmp11
43    %tmp13 = load i32, i32 addrspace(3)* %tmp12, align 4
44    %tmp14 = xor i32 %tmp3, %tmp13
45    %tmp15 = lshr i32 %tmp14, 8
46    %tmp16 = and i32 %tmp15, 16711680
47    %tmp19 = lshr i32 %tmp16, 16
48    %tmp20 = and i32 %tmp19, 255
49    %tmp21 = getelementptr inbounds [256 x i32], [256 x i32] addrspace(3)* @lds1, i32 0, i32 %tmp20
50    %tmp22 = load i32, i32 addrspace(3)* %tmp21, align 4
51    %tmp24 = load i32, i32 addrspace(3)* getelementptr ([256 x i32], [256 x i32] addrspace(3)* @lds2, i32 0, i32 undef), align 4
52    %tmp25 = xor i32 %tmp22, %tmp24
53    %tmp26 = and i32 %tmp25, -16777216
54    %tmp28 = or i32 %0, %tmp26
55    store volatile i32 %tmp28, i32 addrspace(1)* undef
56    ret void
57  }
58
59  attributes #0 = { convergent nounwind }
60  attributes #1 = { convergent nounwind readnone }
61
62  define amdgpu_kernel void @move_waw_hazards() #0 {
63    ret void
64  }
65
66  attributes #0 = { convergent nounwind }
67
68  define amdgpu_kernel void @merge_mmos(i32 addrspace(1)* %ptr_addr1) { ret void }
69  define amdgpu_kernel void @reorder_offsets(i32 addrspace(1)* %reorder_addr1) { ret void }
70
71...
72---
73name:            mem_dependency
74alignment:       1
75exposesReturnsTwice: false
76legalized:       false
77regBankSelected: false
78selected:        false
79tracksRegLiveness: true
80liveins:
81  - { reg: '$vgpr0', virtual-reg: '%1' }
82frameInfo:
83  isFrameAddressTaken: false
84  isReturnAddressTaken: false
85  hasStackMap:     false
86  hasPatchPoint:   false
87  stackSize:       0
88  offsetAdjustment: 0
89  maxAlignment:    0
90  adjustsStack:    false
91  hasCalls:        false
92  maxCallFrameSize: 0
93  hasOpaqueSPAdjustment: false
94  hasVAStart:      false
95  hasMustTailInVarArgFunc: false
96body:             |
97  bb.0:
98    liveins: $vgpr0
99
100    %1:vgpr_32 = COPY $vgpr0
101    $m0 = S_MOV_B32 -1
102    %2:vgpr_32 = DS_READ_B32 %1, 0, 0, implicit $m0, implicit $exec :: (load 4 from %ir.ptr.0)
103    DS_WRITE_B32 %1, killed %2, 64, 0, implicit $m0, implicit $exec :: (store 4 into %ir.ptr.64)
104
105    ; Make this load unmergeable, to tempt SILoadStoreOptimizer into merging the
106    ; other two loads.
107    %6:vreg_64 = DS_READ2_B32 %1, 16, 17, 0, implicit $m0, implicit $exec :: (load 8 from %ir.ptr.64, align 4)
108    %3:vgpr_32 = COPY %6.sub0
109    %4:vgpr_32 = DS_READ_B32 %1, 4, 0, implicit $m0, implicit $exec :: (load 4 from %ir.ptr.4)
110    %5:vgpr_32 = V_ADD_CO_U32_e32 killed %3, killed %4, implicit-def $vcc, implicit $exec
111    DS_WRITE_B32 killed %1, %5, 0, 0, implicit killed $m0, implicit $exec :: (store 4 into %ir.ptr.0)
112    S_ENDPGM 0
113
114...
115---
116# Make sure the asm def isn't moved after the point where it's used for
117# the address.
118# CHECK-LABEL: name: asm_defines_address
119# CHECK: DS_READ2ST64_B32
120# CHECK: DS_READ2ST64_B32
121# CHECK: INLINEASM
122# CHECK: DS_READ_B32
123# CHECK: DS_READ_B32
124name:            asm_defines_address
125tracksRegLiveness: true
126registers:
127  - { id: 0, class: vgpr_32, preferred-register: '' }
128body:             |
129  bb.0:
130    %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
131    %2:vgpr_32 = DS_READ_B32 %1, 3072, 0, implicit $m0, implicit $exec :: (dereferenceable load 4 from `i32 addrspace(3)* getelementptr inbounds ([256 x i32], [256 x i32] addrspace(3)* @lds0, i32 0, i32 0)`, addrspace 3)
132    %3:vgpr_32 = DS_READ_B32 %1, 2048, 0, implicit $m0, implicit $exec :: (load 4 from `i32 addrspace(3)* getelementptr ([256 x i32], [256 x i32] addrspace(3)* @lds1, i32 0, i32 undef)`, addrspace 3)
133    %4:vgpr_32 = DS_READ_B32 %1, 1024, 0, implicit $m0, implicit $exec :: (load 4 from `i32 addrspace(3)* getelementptr ([256 x i32], [256 x i32] addrspace(3)* @lds3, i32 0, i32 undef)`, addrspace 3)
134    INLINEASM &"v_or_b32 $0, 0, $1", 32, 327690, def %0, 327689, %4
135    %5:vgpr_32 = DS_READ_B32 %0, 2048, 0, implicit $m0, implicit $exec :: (load 4 from %ir.tmp12, addrspace 3)
136    %6:vgpr_32 = DS_READ_B32 %5, 2048, 0, implicit $m0, implicit $exec :: (load 4 from %ir.tmp21, addrspace 3)
137    %7:vgpr_32 = DS_READ_B32 %1, 0, 0, implicit $m0, implicit $exec :: (load 4 from `i32 addrspace(3)* getelementptr ([256 x i32], [256 x i32] addrspace(3)* @lds2, i32 0, i32 undef)`, addrspace 3)
138    S_SETPC_B64_return undef $sgpr30_sgpr31, implicit %6, implicit %7
139
140...
141---
142# Make sure Write-after-Write hazards are correctly detected and the
143# instructions moved accordingly.
144# operations.
145# CHECK-LABEL: name: move_waw_hazards
146# CHECK: S_AND_B64
147# CHECK: S_CMP_EQ_U32
148name:            move_waw_hazards
149tracksRegLiveness: true
150body:             |
151  bb.0:
152    liveins: $sgpr0_sgpr1
153
154    %3:sgpr_64 = COPY $sgpr0_sgpr1
155    %6:sreg_32_xm0_xexec = S_MOV_B32 0
156    %7:sreg_32_xm0 = S_MOV_B32 0
157    %8:sreg_64_xexec = REG_SEQUENCE killed %6, %subreg.sub0, %7, %subreg.sub1
158    %9:sgpr_128 = S_LOAD_DWORDX4_IMM killed %8, 0, 0, 0 :: (invariant load 16, addrspace 6)
159    %31:sreg_64_xexec = S_BUFFER_LOAD_DWORDX2_IMM %9, 0, 0, 0 :: (dereferenceable invariant load 4)
160    %10:sreg_32_xm0_xexec = COPY %31.sub0
161    %11:sreg_32_xm0_xexec = COPY killed %31.sub1
162    %12:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM %9, 2, 0, 0 :: (dereferenceable invariant load 4)
163    %13:sreg_64 = V_CMP_NE_U32_e64 killed %11, 0, implicit $exec
164    %15:sreg_64 = V_CMP_NE_U32_e64 killed %12, 0, implicit $exec
165    %17:sreg_64_xexec = S_AND_B64 killed %13, killed %15, implicit-def dead $scc
166    S_CMP_EQ_U32 killed %10, 0, implicit-def $scc
167    %18:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM %9, 3, 0, 0 :: (dereferenceable invariant load 4)
168    S_ENDPGM 0
169...
170---
171# CHECK-LABEL: merge_mmos
172# CHECK: S_BUFFER_LOAD_DWORDX2_IMM %0, 0, 0, 0 :: (dereferenceable invariant load 8, align 4)
173# CHECK: BUFFER_LOAD_DWORDX2_OFFSET %0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 8, align 4)
174# CHECK: BUFFER_STORE_DWORDX2_OFFSET_exact killed %{{[0-9]+}}, %0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8, align 4)
175# CHECK: BUFFER_LOAD_DWORDX2_OFFSET %0, 0, 64, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 8 from %ir.ptr_addr1 + 64, align 4
176# CHECK: BUFFER_STORE_DWORDX2_OFFSET_exact killed %{{[0-9]+}}, %0, 0, 64, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into %ir.ptr_addr1 + 64, align 4
177name: merge_mmos
178tracksRegLiveness: true
179body: |
180  bb.0:
181    liveins: $sgpr0_sgpr1_sgpr2_sgpr3
182
183    %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
184    %1:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM %0, 0, 0, 0 :: (dereferenceable invariant load 4)
185    %2:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM %0, 1, 0, 0 :: (dereferenceable invariant load 4)
186    %3:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4)
187    %4:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %0, 0, 4, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4)
188    BUFFER_STORE_DWORD_OFFSET_exact %3, %0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4)
189    BUFFER_STORE_DWORD_OFFSET_exact %4, %0, 0, 4, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4)
190    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %0, 0, 64, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4 from %ir.ptr_addr1 + 64)
191    %6:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %0, 0, 68, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4 from %ir.ptr_addr1 + 68)
192    BUFFER_STORE_DWORD_OFFSET_exact %5, %0, 0, 64, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into %ir.ptr_addr1 + 64)
193    BUFFER_STORE_DWORD_OFFSET_exact %6, %0, 0, 68, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into %ir.ptr_addr1 + 68)
194
195    S_ENDPGM 0
196
197...
198---
199# CHECK-LABEL: reorder_offsets
200# CHECK-DAG: BUFFER_STORE_DWORDX2_OFFSET_exact killed %{{[0-9]+}}, %0, 0, 16, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into %ir.reorder_addr1 + 16, align 4, addrspace 1)
201# CHECK-DAG: BUFFER_STORE_DWORDX4_OFFSET_exact killed %{{[0-9]+}}, %0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into %ir.reorder_addr1, align 4, addrspace 1)
202
203name: reorder_offsets
204tracksRegLiveness: true
205body: |
206  bb.0:
207    liveins: $sgpr0_sgpr1_sgpr2_sgpr3
208
209    %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
210    %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
211    BUFFER_STORE_DWORD_OFFSET_exact %1, %0, 0, 4, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into %ir.reorder_addr1 + 4)
212    BUFFER_STORE_DWORD_OFFSET_exact %1, %0, 0, 8, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into %ir.reorder_addr1 + 8)
213    BUFFER_STORE_DWORD_OFFSET_exact %1, %0, 0, 12, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into %ir.reorder_addr1 + 12)
214    BUFFER_STORE_DWORD_OFFSET_exact %1, %0, 0, 16, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into %ir.reorder_addr1 + 16)
215    BUFFER_STORE_DWORD_OFFSET_exact %1, %0, 0, 20, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into %ir.reorder_addr1 + 20)
216    BUFFER_STORE_DWORD_OFFSET_exact %1, %0, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into %ir.reorder_addr1)
217    S_ENDPGM 0
218
219
220...
221