1; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -debug-only=machine-scheduler < %s 2> %t | FileCheck --enable-var-scope --check-prefixes=CHECK,GCN %s
2; RUN: FileCheck --enable-var-scope --check-prefixes=CHECK,DBG %s < %t
3; REQUIRES: asserts
4
5; CHECK-LABEL: {{^}}cluster_load_cluster_store:
6define amdgpu_kernel void @cluster_load_cluster_store(i32* noalias %lb, i32* noalias %sb) {
7bb:
8
9; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 8
10; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 8
11; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
12; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
13; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
14; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
15
16; DBG: Cluster ld/st SU(1) - SU(2)
17
18; DBG: Cluster ld/st SU([[L1:[0-9]+]]) - SU([[L2:[0-9]+]])
19; DBG: Cluster ld/st SU([[L2]]) - SU([[L3:[0-9]+]])
20; DBG: Cluster ld/st SU([[L3]]) - SU([[L4:[0-9]+]])
21; GCN:      flat_load_dword [[LD1:v[0-9]+]], v[{{[0-9:]+}}]
22; GCN-NEXT: flat_load_dword [[LD2:v[0-9]+]], v[{{[0-9:]+}}] offset:8
23; GCN-NEXT: flat_load_dword [[LD3:v[0-9]+]], v[{{[0-9:]+}}] offset:16
24; GCN-NEXT: flat_load_dword [[LD4:v[0-9]+]], v[{{[0-9:]+}}] offset:24
25  %la0 = getelementptr inbounds i32, i32* %lb, i32 0
26  %ld0 = load i32, i32* %la0
27  %la1 = getelementptr inbounds i32, i32* %lb, i32 2
28  %ld1 = load i32, i32* %la1
29  %la2 = getelementptr inbounds i32, i32* %lb, i32 4
30  %ld2 = load i32, i32* %la2
31  %la3 = getelementptr inbounds i32, i32* %lb, i32 6
32  %ld3 = load i32, i32* %la3
33
34; DBG: Cluster ld/st SU([[S1:[0-9]+]]) - SU([[S2:[0-9]+]])
35; DBG: Cluster ld/st SU([[S2]]) - SU([[S3:[0-9]+]])
36; DBG: Cluster ld/st SU([[S3]]) - SU([[S4:[0-9]+]])
37; GCN:      flat_store_dword v[{{[0-9:]+}}], [[LD1]]
38; GCN-NEXT: flat_store_dword v[{{[0-9:]+}}], [[LD2]] offset:8
39; GCN-NEXT: flat_store_dword v[{{[0-9:]+}}], [[LD3]] offset:16
40; GCN-NEXT: flat_store_dword v[{{[0-9:]+}}], [[LD4]] offset:24
41  %sa0 = getelementptr inbounds i32, i32* %sb, i32 0
42  store i32 %ld0, i32* %sa0
43  %sa1 = getelementptr inbounds i32, i32* %sb, i32 2
44  store i32 %ld1, i32* %sa1
45  %sa2 = getelementptr inbounds i32, i32* %sb, i32 4
46  store i32 %ld2, i32* %sa2
47  %sa3 = getelementptr inbounds i32, i32* %sb, i32 6
48  store i32 %ld3, i32* %sa3
49
50  ret void
51}
52
53; CHECK-LABEL: {{^}}cluster_load_valu_cluster_store:
54define amdgpu_kernel void @cluster_load_valu_cluster_store(i32* noalias %lb, i32* noalias %sb) {
55bb:
56; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 8
57; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 8
58; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
59; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
60; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
61; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 4
62
63; DBG: Cluster ld/st SU(1) - SU(2)
64
65; DBG: Cluster ld/st SU([[L1:[0-9]+]]) - SU([[L2:[0-9]+]])
66; DBG: Cluster ld/st SU([[L2]]) - SU([[L3:[0-9]+]])
67; DBG: Cluster ld/st SU([[L3]]) - SU([[L4:[0-9]+]])
68; GCN:      flat_load_dword [[LD1:v[0-9]+]], v[{{[0-9:]+}}]
69; GCN-NEXT: flat_load_dword [[LD2:v[0-9]+]], v[{{[0-9:]+}}] offset:8
70; GCN-NEXT: flat_load_dword [[LD3:v[0-9]+]], v[{{[0-9:]+}}] offset:16
71; GCN-NEXT: flat_load_dword [[LD4:v[0-9]+]], v[{{[0-9:]+}}] offset:24
72  %la0 = getelementptr inbounds i32, i32* %lb, i32 0
73  %ld0 = load i32, i32* %la0
74  %la1 = getelementptr inbounds i32, i32* %lb, i32 2
75  %ld1 = load i32, i32* %la1
76  %la2 = getelementptr inbounds i32, i32* %lb, i32 4
77  %ld2 = load i32, i32* %la2
78  %la3 = getelementptr inbounds i32, i32* %lb, i32 6
79  %ld3 = load i32, i32* %la3
80
81; DBG: Cluster ld/st SU([[S1:[0-9]+]]) - SU([[S2:[0-9]+]])
82; DBG: Cluster ld/st SU([[S2]]) - SU([[S3:[0-9]+]])
83; DBG: Cluster ld/st SU([[S3]]) - SU([[S4:[0-9]+]])
84; GCN:      v_add_u32_e32 [[ST2:v[0-9]+]], 1, [[LD2]]
85; GCN:      flat_store_dword v[{{[0-9:]+}}], [[LD1]]
86; GCN-NEXT: flat_store_dword v[{{[0-9:]+}}], [[ST2]] offset:8
87; GCN-NEXT: flat_store_dword v[{{[0-9:]+}}], [[LD3]] offset:16
88; GCN-NEXT: flat_store_dword v[{{[0-9:]+}}], [[LD4]] offset:24
89  %sa0 = getelementptr inbounds i32, i32* %sb, i32 0
90  store i32 %ld0, i32* %sa0
91  %sa1 = getelementptr inbounds i32, i32* %sb, i32 2
92  %add = add i32 %ld1, 1
93  store i32 %add, i32* %sa1
94  %sa2 = getelementptr inbounds i32, i32* %sb, i32 4
95  store i32 %ld2, i32* %sa2
96  %sa3 = getelementptr inbounds i32, i32* %sb, i32 6
97  store i32 %ld3, i32* %sa3
98
99  ret void
100}
101
102; Cluster loads from the same texture with different coordinates
103; CHECK-LABEL: {{^}}cluster_image_load:
104; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
105; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
106; DBG: {{^}}Cluster ld/st [[SU1:SU\([0-9]+\)]] - [[SU2:SU\([0-9]+\)]]
107; DBG: {{^}}[[SU1]]: {{.*}} IMAGE_LOAD
108; DBG: {{^}}[[SU2]]: {{.*}} IMAGE_LOAD
109; GCN:      image_load v
110; GCN-NEXT: image_load v
111define amdgpu_ps void @cluster_image_load(<8 x i32> inreg %src, <8 x i32> inreg %dst, i32 %x, i32 %y) {
112entry:
113  %x1 = add i32 %x, 1
114  %y1 = add i32 %y, 1
115  %val1 = call <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32 15, i32 %x1, i32 %y1, i32 0, <8 x i32> %src, i32 0, i32 0)
116  %x2 = add i32 %x, 2
117  %y2 = add i32 %y, 2
118  %val2 = call <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32 15, i32 %x2, i32 %y2, i32 0, <8 x i32> %src, i32 0, i32 0)
119  %val = fadd fast <4 x float> %val1, %val2
120  call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %val, i32 15, i32 %x, i32 %y, <8 x i32> %dst, i32 0, i32 0)
121  ret void
122}
123
124; Don't cluster loads from different textures
125; CHECK-LABEL: {{^}}no_cluster_image_load:
126; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
127; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
128; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
129; DBG-NOT: {{^}}Cluster ld/st
130define amdgpu_ps void @no_cluster_image_load(<8 x i32> inreg %src1, <8 x i32> inreg %src2, <8 x i32> inreg %dst, i32 %x, i32 %y) {
131entry:
132  %val1 = call <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32 15, i32 %x, i32 %y, i32 0, <8 x i32> %src1, i32 0, i32 0)
133  %val2 = call <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32 15, i32 %x, i32 %y, i32 0, <8 x i32> %src2, i32 0, i32 0)
134  %val = fadd fast <4 x float> %val1, %val2
135  call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %val, i32 15, i32 %x, i32 %y, <8 x i32> %dst, i32 0, i32 0)
136  ret void
137}
138
139; Cluster loads from the same texture and sampler with different coordinates
140; CHECK-LABEL: {{^}}cluster_image_sample:
141; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
142; DBG: Num BaseOps: {{[1-9]+}}, Offset: {{[0-9]+}}, OffsetIsScalable: {{[01]}}, Width: 16
143; DBG: {{^}}Cluster ld/st [[SU1:SU\([0-9]+\)]] - [[SU2:SU\([0-9]+\)]]
144; DBG: {{^}}[[SU1]]: {{.*}} IMAGE_SAMPLE
145; DBG: {{^}}[[SU2]]: {{.*}} IMAGE_SAMPLE
146; GCN:      image_sample_d
147; GCN-NEXT: image_sample_d
148define amdgpu_ps void @cluster_image_sample(<8 x i32> inreg %src, <4 x i32> inreg %smp, <8 x i32> inreg %dst, i32 %x, i32 %y) {
149entry:
150  %s = sitofp i32 %x to float
151  %t = sitofp i32 %y to float
152  %s1 = fadd float %s, 1.0
153  %t1 = fadd float %t, 1.0
154  %val1 = call <4 x float> @llvm.amdgcn.image.sample.d.2d.v4f32.f32(i32 15, float %s1, float %t1, float 0.0, float 0.0, float 0.0, float 0.0, <8 x i32> %src, <4 x i32> %smp, i1 false, i32 0, i32 0)
155  %s2 = fadd float %s, 2.0
156  %t2 = fadd float %t, 2.0
157  %val2 = call <4 x float> @llvm.amdgcn.image.sample.d.2d.v4f32.f32(i32 15, float %s2, float %t2, float 1.0, float 1.0, float 1.0, float 1.0, <8 x i32> %src, <4 x i32> %smp, i1 false, i32 0, i32 0)
158  %val = fadd fast <4 x float> %val1, %val2
159  call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %val, i32 15, i32 %x, i32 %y, <8 x i32> %dst, i32 0, i32 0)
160  ret void
161}
162
163declare <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32 immarg, i32, i32, i32, <8 x i32>, i32 immarg, i32 immarg)
164declare <4 x float> @llvm.amdgcn.image.sample.d.2d.v4f32.f32(i32, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32)
165declare void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float>, i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg)
166