1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs | FileCheck %s -check-prefixes=FUNC,GCN,SICIVI,SI
3; RUN: llc < %s -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s -check-prefixes=FUNC,GCN,SICIVI,VI
4; RUN: llc < %s -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -verify-machineinstrs | FileCheck %s -check-prefixes=FUNC,GCN,GFX9
5
6
7declare { i32, i1 } @llvm.sadd.with.overflow.i32(i32, i32) nounwind readnone
8declare { i64, i1 } @llvm.sadd.with.overflow.i64(i64, i64) nounwind readnone
9
10
11declare { <2 x i32>, <2 x i1> } @llvm.sadd.with.overflow.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
12
13define amdgpu_kernel void @saddo_i64_zext(i64 addrspace(1)* %out, i64 %a, i64 %b) nounwind {
14; SI-LABEL: saddo_i64_zext:
15; SI:       ; %bb.0:
16; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
17; SI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
18; SI-NEXT:    s_mov_b32 s3, 0xf000
19; SI-NEXT:    s_mov_b32 s2, -1
20; SI-NEXT:    s_waitcnt lgkmcnt(0)
21; SI-NEXT:    v_mov_b32_e32 v0, s6
22; SI-NEXT:    s_add_u32 s10, s6, s8
23; SI-NEXT:    s_addc_u32 s11, s7, s9
24; SI-NEXT:    v_mov_b32_e32 v1, s7
25; SI-NEXT:    v_cmp_lt_i64_e32 vcc, s[10:11], v[0:1]
26; SI-NEXT:    v_cmp_lt_i64_e64 s[6:7], s[8:9], 0
27; SI-NEXT:    s_mov_b32 s0, s4
28; SI-NEXT:    s_mov_b32 s1, s5
29; SI-NEXT:    s_xor_b64 s[4:5], s[6:7], vcc
30; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
31; SI-NEXT:    v_mov_b32_e32 v1, s11
32; SI-NEXT:    v_add_i32_e32 v0, vcc, s10, v0
33; SI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
34; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
35; SI-NEXT:    s_endpgm
36;
37; VI-LABEL: saddo_i64_zext:
38; VI:       ; %bb.0:
39; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
40; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
41; VI-NEXT:    s_waitcnt lgkmcnt(0)
42; VI-NEXT:    v_mov_b32_e32 v1, s6
43; VI-NEXT:    s_add_u32 s2, s6, s0
44; VI-NEXT:    s_addc_u32 s3, s7, s1
45; VI-NEXT:    v_mov_b32_e32 v2, s7
46; VI-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[1:2]
47; VI-NEXT:    v_cmp_lt_i64_e64 s[8:9], s[0:1], 0
48; VI-NEXT:    v_mov_b32_e32 v3, s3
49; VI-NEXT:    s_xor_b64 s[0:1], s[8:9], vcc
50; VI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
51; VI-NEXT:    v_add_u32_e32 v2, vcc, s2, v2
52; VI-NEXT:    v_mov_b32_e32 v0, s4
53; VI-NEXT:    v_mov_b32_e32 v1, s5
54; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
55; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
56; VI-NEXT:    s_endpgm
57;
58; GFX9-LABEL: saddo_i64_zext:
59; GFX9:       ; %bb.0:
60; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
61; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
62; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
63; GFX9-NEXT:    v_mov_b32_e32 v1, s6
64; GFX9-NEXT:    s_add_u32 s2, s6, s0
65; GFX9-NEXT:    s_addc_u32 s3, s7, s1
66; GFX9-NEXT:    v_mov_b32_e32 v2, s7
67; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[1:2]
68; GFX9-NEXT:    v_cmp_lt_i64_e64 s[8:9], s[0:1], 0
69; GFX9-NEXT:    v_mov_b32_e32 v3, s3
70; GFX9-NEXT:    s_xor_b64 s[0:1], s[8:9], vcc
71; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
72; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s2, v2
73; GFX9-NEXT:    v_mov_b32_e32 v0, s4
74; GFX9-NEXT:    v_mov_b32_e32 v1, s5
75; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
76; GFX9-NEXT:    global_store_dwordx2 v[0:1], v[2:3], off
77; GFX9-NEXT:    s_endpgm
78  %sadd = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 %a, i64 %b) nounwind
79  %val = extractvalue { i64, i1 } %sadd, 0
80  %carry = extractvalue { i64, i1 } %sadd, 1
81  %ext = zext i1 %carry to i64
82  %add2 = add i64 %val, %ext
83  store i64 %add2, i64 addrspace(1)* %out, align 8
84  ret void
85}
86
87define amdgpu_kernel void @s_saddo_i32(i32 addrspace(1)* %out, i1 addrspace(1)* %carryout, i32 %a, i32 %b) nounwind {
88; SI-LABEL: s_saddo_i32:
89; SI:       ; %bb.0:
90; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
91; SI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
92; SI-NEXT:    s_mov_b32 s3, 0xf000
93; SI-NEXT:    s_mov_b32 s2, -1
94; SI-NEXT:    s_waitcnt lgkmcnt(0)
95; SI-NEXT:    s_mov_b32 s0, s4
96; SI-NEXT:    v_cmp_lt_i32_e64 s[10:11], s9, 0
97; SI-NEXT:    s_add_i32 s9, s8, s9
98; SI-NEXT:    v_mov_b32_e32 v0, s8
99; SI-NEXT:    s_mov_b32 s1, s5
100; SI-NEXT:    v_cmp_lt_i32_e32 vcc, s9, v0
101; SI-NEXT:    v_mov_b32_e32 v0, s9
102; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
103; SI-NEXT:    s_xor_b64 s[0:1], s[10:11], vcc
104; SI-NEXT:    s_mov_b32 s4, s6
105; SI-NEXT:    s_mov_b32 s5, s7
106; SI-NEXT:    s_mov_b32 s6, s2
107; SI-NEXT:    s_mov_b32 s7, s3
108; SI-NEXT:    s_waitcnt expcnt(0)
109; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
110; SI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
111; SI-NEXT:    s_endpgm
112;
113; VI-LABEL: s_saddo_i32:
114; VI:       ; %bb.0:
115; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
116; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
117; VI-NEXT:    s_waitcnt lgkmcnt(0)
118; VI-NEXT:    v_mov_b32_e32 v0, s4
119; VI-NEXT:    v_cmp_lt_i32_e64 s[2:3], s1, 0
120; VI-NEXT:    s_add_i32 s1, s0, s1
121; VI-NEXT:    v_mov_b32_e32 v4, s0
122; VI-NEXT:    v_cmp_lt_i32_e32 vcc, s1, v4
123; VI-NEXT:    v_mov_b32_e32 v4, s1
124; VI-NEXT:    v_mov_b32_e32 v1, s5
125; VI-NEXT:    s_xor_b64 s[0:1], s[2:3], vcc
126; VI-NEXT:    flat_store_dword v[0:1], v4
127; VI-NEXT:    v_mov_b32_e32 v2, s6
128; VI-NEXT:    v_mov_b32_e32 v3, s7
129; VI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
130; VI-NEXT:    flat_store_byte v[2:3], v0
131; VI-NEXT:    s_endpgm
132;
133; GFX9-LABEL: s_saddo_i32:
134; GFX9:       ; %bb.0:
135; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
136; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
137; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
138; GFX9-NEXT:    v_mov_b32_e32 v0, s4
139; GFX9-NEXT:    v_cmp_lt_i32_e64 s[2:3], s1, 0
140; GFX9-NEXT:    s_add_i32 s1, s0, s1
141; GFX9-NEXT:    v_mov_b32_e32 v4, s0
142; GFX9-NEXT:    v_cmp_lt_i32_e32 vcc, s1, v4
143; GFX9-NEXT:    v_mov_b32_e32 v4, s1
144; GFX9-NEXT:    v_mov_b32_e32 v1, s5
145; GFX9-NEXT:    s_xor_b64 s[0:1], s[2:3], vcc
146; GFX9-NEXT:    global_store_dword v[0:1], v4, off
147; GFX9-NEXT:    v_mov_b32_e32 v2, s6
148; GFX9-NEXT:    v_mov_b32_e32 v3, s7
149; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
150; GFX9-NEXT:    global_store_byte v[2:3], v0, off
151; GFX9-NEXT:    s_endpgm
152  %sadd = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a, i32 %b) nounwind
153  %val = extractvalue { i32, i1 } %sadd, 0
154  %carry = extractvalue { i32, i1 } %sadd, 1
155  store i32 %val, i32 addrspace(1)* %out, align 4
156  store i1 %carry, i1 addrspace(1)* %carryout
157  ret void
158}
159
160define amdgpu_kernel void @v_saddo_i32(i32 addrspace(1)* %out, i1 addrspace(1)* %carryout, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
161; SI-LABEL: v_saddo_i32:
162; SI:       ; %bb.0:
163; SI-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x9
164; SI-NEXT:    s_mov_b32 s15, 0xf000
165; SI-NEXT:    s_mov_b32 s14, -1
166; SI-NEXT:    s_mov_b32 s2, s14
167; SI-NEXT:    s_mov_b32 s3, s15
168; SI-NEXT:    s_waitcnt lgkmcnt(0)
169; SI-NEXT:    s_mov_b32 s0, s10
170; SI-NEXT:    s_mov_b32 s1, s11
171; SI-NEXT:    s_mov_b32 s10, s14
172; SI-NEXT:    s_mov_b32 s11, s15
173; SI-NEXT:    buffer_load_dword v0, off, s[8:11], 0
174; SI-NEXT:    buffer_load_dword v1, off, s[0:3], 0
175; SI-NEXT:    s_mov_b32 s12, s6
176; SI-NEXT:    s_mov_b32 s13, s7
177; SI-NEXT:    s_mov_b32 s6, s14
178; SI-NEXT:    s_mov_b32 s7, s15
179; SI-NEXT:    s_waitcnt vmcnt(0)
180; SI-NEXT:    v_add_i32_e32 v2, vcc, v1, v0
181; SI-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v1
182; SI-NEXT:    v_cmp_lt_i32_e64 s[0:1], v2, v0
183; SI-NEXT:    s_xor_b64 s[0:1], vcc, s[0:1]
184; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
185; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 0
186; SI-NEXT:    buffer_store_byte v0, off, s[12:15], 0
187; SI-NEXT:    s_endpgm
188;
189; VI-LABEL: v_saddo_i32:
190; VI:       ; %bb.0:
191; VI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
192; VI-NEXT:    s_waitcnt lgkmcnt(0)
193; VI-NEXT:    v_mov_b32_e32 v4, s6
194; VI-NEXT:    v_mov_b32_e32 v6, s4
195; VI-NEXT:    v_mov_b32_e32 v7, s5
196; VI-NEXT:    v_mov_b32_e32 v5, s7
197; VI-NEXT:    flat_load_dword v6, v[6:7]
198; VI-NEXT:    flat_load_dword v4, v[4:5]
199; VI-NEXT:    v_mov_b32_e32 v2, s0
200; VI-NEXT:    v_mov_b32_e32 v3, s1
201; VI-NEXT:    v_mov_b32_e32 v0, s2
202; VI-NEXT:    v_mov_b32_e32 v1, s3
203; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
204; VI-NEXT:    v_add_u32_e32 v5, vcc, v4, v6
205; VI-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v4
206; VI-NEXT:    v_cmp_lt_i32_e64 s[0:1], v5, v6
207; VI-NEXT:    s_xor_b64 s[0:1], vcc, s[0:1]
208; VI-NEXT:    flat_store_dword v[2:3], v5
209; VI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
210; VI-NEXT:    flat_store_byte v[0:1], v2
211; VI-NEXT:    s_endpgm
212;
213; GFX9-LABEL: v_saddo_i32:
214; GFX9:       ; %bb.0:
215; GFX9-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
216; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
217; GFX9-NEXT:    v_mov_b32_e32 v4, s6
218; GFX9-NEXT:    v_mov_b32_e32 v6, s4
219; GFX9-NEXT:    v_mov_b32_e32 v7, s5
220; GFX9-NEXT:    v_mov_b32_e32 v5, s7
221; GFX9-NEXT:    global_load_dword v6, v[6:7], off
222; GFX9-NEXT:    global_load_dword v4, v[4:5], off
223; GFX9-NEXT:    v_mov_b32_e32 v2, s0
224; GFX9-NEXT:    v_mov_b32_e32 v3, s1
225; GFX9-NEXT:    v_mov_b32_e32 v0, s2
226; GFX9-NEXT:    v_mov_b32_e32 v1, s3
227; GFX9-NEXT:    s_waitcnt vmcnt(0)
228; GFX9-NEXT:    v_add_u32_e32 v5, v6, v4
229; GFX9-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v4
230; GFX9-NEXT:    v_cmp_lt_i32_e64 s[0:1], v5, v6
231; GFX9-NEXT:    s_xor_b64 s[0:1], vcc, s[0:1]
232; GFX9-NEXT:    global_store_dword v[2:3], v5, off
233; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
234; GFX9-NEXT:    global_store_byte v[0:1], v2, off
235; GFX9-NEXT:    s_endpgm
236  %a = load i32, i32 addrspace(1)* %aptr, align 4
237  %b = load i32, i32 addrspace(1)* %bptr, align 4
238  %sadd = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a, i32 %b) nounwind
239  %val = extractvalue { i32, i1 } %sadd, 0
240  %carry = extractvalue { i32, i1 } %sadd, 1
241  store i32 %val, i32 addrspace(1)* %out, align 4
242  store i1 %carry, i1 addrspace(1)* %carryout
243  ret void
244}
245
246define amdgpu_kernel void @s_saddo_i64(i64 addrspace(1)* %out, i1 addrspace(1)* %carryout, i64 %a, i64 %b) nounwind {
247; SI-LABEL: s_saddo_i64:
248; SI:       ; %bb.0:
249; SI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x9
250; SI-NEXT:    s_mov_b32 s11, 0xf000
251; SI-NEXT:    s_mov_b32 s10, -1
252; SI-NEXT:    s_waitcnt lgkmcnt(0)
253; SI-NEXT:    s_add_u32 s12, s4, s6
254; SI-NEXT:    v_mov_b32_e32 v0, s4
255; SI-NEXT:    s_addc_u32 s13, s5, s7
256; SI-NEXT:    v_mov_b32_e32 v1, s5
257; SI-NEXT:    v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1]
258; SI-NEXT:    v_mov_b32_e32 v0, s12
259; SI-NEXT:    v_cmp_lt_i64_e64 s[4:5], s[6:7], 0
260; SI-NEXT:    s_mov_b32 s8, s2
261; SI-NEXT:    s_mov_b32 s9, s3
262; SI-NEXT:    s_mov_b32 s2, s10
263; SI-NEXT:    s_mov_b32 s3, s11
264; SI-NEXT:    v_mov_b32_e32 v1, s13
265; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
266; SI-NEXT:    s_xor_b64 s[0:1], s[4:5], vcc
267; SI-NEXT:    s_waitcnt expcnt(0)
268; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
269; SI-NEXT:    buffer_store_byte v0, off, s[8:11], 0
270; SI-NEXT:    s_endpgm
271;
272; VI-LABEL: s_saddo_i64:
273; VI:       ; %bb.0:
274; VI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
275; VI-NEXT:    s_waitcnt lgkmcnt(0)
276; VI-NEXT:    v_mov_b32_e32 v2, s0
277; VI-NEXT:    v_mov_b32_e32 v4, s4
278; VI-NEXT:    s_add_u32 s0, s4, s6
279; VI-NEXT:    v_mov_b32_e32 v3, s1
280; VI-NEXT:    s_addc_u32 s1, s5, s7
281; VI-NEXT:    v_mov_b32_e32 v5, s5
282; VI-NEXT:    v_cmp_lt_i64_e32 vcc, s[0:1], v[4:5]
283; VI-NEXT:    v_mov_b32_e32 v0, s2
284; VI-NEXT:    v_mov_b32_e32 v1, s3
285; VI-NEXT:    v_cmp_lt_i64_e64 s[2:3], s[6:7], 0
286; VI-NEXT:    v_mov_b32_e32 v5, s1
287; VI-NEXT:    v_mov_b32_e32 v4, s0
288; VI-NEXT:    s_xor_b64 s[0:1], s[2:3], vcc
289; VI-NEXT:    flat_store_dwordx2 v[2:3], v[4:5]
290; VI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
291; VI-NEXT:    flat_store_byte v[0:1], v2
292; VI-NEXT:    s_endpgm
293;
294; GFX9-LABEL: s_saddo_i64:
295; GFX9:       ; %bb.0:
296; GFX9-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
297; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
298; GFX9-NEXT:    v_mov_b32_e32 v2, s0
299; GFX9-NEXT:    v_mov_b32_e32 v4, s4
300; GFX9-NEXT:    s_add_u32 s0, s4, s6
301; GFX9-NEXT:    v_mov_b32_e32 v3, s1
302; GFX9-NEXT:    s_addc_u32 s1, s5, s7
303; GFX9-NEXT:    v_mov_b32_e32 v5, s5
304; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, s[0:1], v[4:5]
305; GFX9-NEXT:    v_mov_b32_e32 v0, s2
306; GFX9-NEXT:    v_mov_b32_e32 v1, s3
307; GFX9-NEXT:    v_cmp_lt_i64_e64 s[2:3], s[6:7], 0
308; GFX9-NEXT:    v_mov_b32_e32 v5, s1
309; GFX9-NEXT:    v_mov_b32_e32 v4, s0
310; GFX9-NEXT:    s_xor_b64 s[0:1], s[2:3], vcc
311; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[4:5], off
312; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
313; GFX9-NEXT:    global_store_byte v[0:1], v2, off
314; GFX9-NEXT:    s_endpgm
315  %sadd = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 %a, i64 %b) nounwind
316  %val = extractvalue { i64, i1 } %sadd, 0
317  %carry = extractvalue { i64, i1 } %sadd, 1
318  store i64 %val, i64 addrspace(1)* %out, align 8
319  store i1 %carry, i1 addrspace(1)* %carryout
320  ret void
321}
322
323define amdgpu_kernel void @v_saddo_i64(i64 addrspace(1)* %out, i1 addrspace(1)* %carryout, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr) nounwind {
324; SI-LABEL: v_saddo_i64:
325; SI:       ; %bb.0:
326; SI-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x9
327; SI-NEXT:    s_mov_b32 s15, 0xf000
328; SI-NEXT:    s_mov_b32 s14, -1
329; SI-NEXT:    s_mov_b32 s2, s14
330; SI-NEXT:    s_mov_b32 s3, s15
331; SI-NEXT:    s_waitcnt lgkmcnt(0)
332; SI-NEXT:    s_mov_b32 s0, s10
333; SI-NEXT:    s_mov_b32 s1, s11
334; SI-NEXT:    s_mov_b32 s10, s14
335; SI-NEXT:    s_mov_b32 s11, s15
336; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
337; SI-NEXT:    buffer_load_dwordx2 v[2:3], off, s[0:3], 0
338; SI-NEXT:    s_mov_b32 s12, s6
339; SI-NEXT:    s_mov_b32 s13, s7
340; SI-NEXT:    s_mov_b32 s6, s14
341; SI-NEXT:    s_mov_b32 s7, s15
342; SI-NEXT:    s_waitcnt vmcnt(0)
343; SI-NEXT:    v_add_i32_e32 v4, vcc, v0, v2
344; SI-NEXT:    v_addc_u32_e32 v5, vcc, v1, v3, vcc
345; SI-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[2:3]
346; SI-NEXT:    v_cmp_lt_i64_e64 s[0:1], v[4:5], v[0:1]
347; SI-NEXT:    buffer_store_dwordx2 v[4:5], off, s[4:7], 0
348; SI-NEXT:    s_xor_b64 s[0:1], vcc, s[0:1]
349; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
350; SI-NEXT:    buffer_store_byte v0, off, s[12:15], 0
351; SI-NEXT:    s_endpgm
352;
353; VI-LABEL: v_saddo_i64:
354; VI:       ; %bb.0:
355; VI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
356; VI-NEXT:    s_waitcnt lgkmcnt(0)
357; VI-NEXT:    v_mov_b32_e32 v4, s6
358; VI-NEXT:    v_mov_b32_e32 v5, s7
359; VI-NEXT:    v_mov_b32_e32 v6, s4
360; VI-NEXT:    v_mov_b32_e32 v7, s5
361; VI-NEXT:    flat_load_dwordx2 v[6:7], v[6:7]
362; VI-NEXT:    flat_load_dwordx2 v[4:5], v[4:5]
363; VI-NEXT:    v_mov_b32_e32 v2, s0
364; VI-NEXT:    v_mov_b32_e32 v3, s1
365; VI-NEXT:    v_mov_b32_e32 v0, s2
366; VI-NEXT:    v_mov_b32_e32 v1, s3
367; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
368; VI-NEXT:    v_add_u32_e32 v8, vcc, v6, v4
369; VI-NEXT:    v_addc_u32_e32 v9, vcc, v7, v5, vcc
370; VI-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[4:5]
371; VI-NEXT:    v_cmp_lt_i64_e64 s[0:1], v[8:9], v[6:7]
372; VI-NEXT:    flat_store_dwordx2 v[2:3], v[8:9]
373; VI-NEXT:    s_xor_b64 s[0:1], vcc, s[0:1]
374; VI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
375; VI-NEXT:    flat_store_byte v[0:1], v2
376; VI-NEXT:    s_endpgm
377;
378; GFX9-LABEL: v_saddo_i64:
379; GFX9:       ; %bb.0:
380; GFX9-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
381; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
382; GFX9-NEXT:    v_mov_b32_e32 v4, s6
383; GFX9-NEXT:    v_mov_b32_e32 v5, s7
384; GFX9-NEXT:    v_mov_b32_e32 v6, s4
385; GFX9-NEXT:    v_mov_b32_e32 v7, s5
386; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[6:7], off
387; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[4:5], off
388; GFX9-NEXT:    v_mov_b32_e32 v2, s0
389; GFX9-NEXT:    v_mov_b32_e32 v3, s1
390; GFX9-NEXT:    v_mov_b32_e32 v0, s2
391; GFX9-NEXT:    v_mov_b32_e32 v1, s3
392; GFX9-NEXT:    s_waitcnt vmcnt(0)
393; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v6, v4
394; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, v7, v5, vcc
395; GFX9-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[4:5]
396; GFX9-NEXT:    v_cmp_lt_i64_e64 s[0:1], v[8:9], v[6:7]
397; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[8:9], off
398; GFX9-NEXT:    s_xor_b64 s[0:1], vcc, s[0:1]
399; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
400; GFX9-NEXT:    global_store_byte v[0:1], v2, off
401; GFX9-NEXT:    s_endpgm
402  %a = load i64, i64 addrspace(1)* %aptr, align 4
403  %b = load i64, i64 addrspace(1)* %bptr, align 4
404  %sadd = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 %a, i64 %b) nounwind
405  %val = extractvalue { i64, i1 } %sadd, 0
406  %carry = extractvalue { i64, i1 } %sadd, 1
407  store i64 %val, i64 addrspace(1)* %out, align 8
408  store i1 %carry, i1 addrspace(1)* %carryout
409  ret void
410}
411
412define amdgpu_kernel void @v_saddo_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %carryout, <2 x i32> addrspace(1)* %aptr, <2 x i32> addrspace(1)* %bptr) nounwind {
413; SI-LABEL: v_saddo_v2i32:
414; SI:       ; %bb.0:
415; SI-NEXT:    s_load_dwordx8 s[8:15], s[0:1], 0x9
416; SI-NEXT:    s_mov_b32 s19, 0xf000
417; SI-NEXT:    s_mov_b32 s18, -1
418; SI-NEXT:    s_mov_b32 s2, s18
419; SI-NEXT:    s_mov_b32 s3, s19
420; SI-NEXT:    s_waitcnt lgkmcnt(0)
421; SI-NEXT:    s_mov_b32 s0, s14
422; SI-NEXT:    s_mov_b32 s1, s15
423; SI-NEXT:    s_mov_b32 s14, s18
424; SI-NEXT:    s_mov_b32 s15, s19
425; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[12:15], 0
426; SI-NEXT:    buffer_load_dwordx2 v[2:3], off, s[0:3], 0
427; SI-NEXT:    s_mov_b32 s16, s10
428; SI-NEXT:    s_mov_b32 s17, s11
429; SI-NEXT:    s_mov_b32 s10, s18
430; SI-NEXT:    s_mov_b32 s11, s19
431; SI-NEXT:    s_waitcnt vmcnt(0)
432; SI-NEXT:    v_add_i32_e32 v5, vcc, v1, v3
433; SI-NEXT:    v_add_i32_e32 v4, vcc, v0, v2
434; SI-NEXT:    v_cmp_gt_i32_e64 s[0:1], 0, v3
435; SI-NEXT:    v_cmp_lt_i32_e64 s[4:5], v5, v1
436; SI-NEXT:    s_xor_b64 s[0:1], s[0:1], s[4:5]
437; SI-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v2
438; SI-NEXT:    v_cmp_lt_i32_e64 s[2:3], v4, v0
439; SI-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
440; SI-NEXT:    s_xor_b64 s[0:1], vcc, s[2:3]
441; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
442; SI-NEXT:    buffer_store_dwordx2 v[4:5], off, s[8:11], 0
443; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[16:19], 0
444; SI-NEXT:    s_endpgm
445;
446; VI-LABEL: v_saddo_v2i32:
447; VI:       ; %bb.0:
448; VI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
449; VI-NEXT:    s_waitcnt lgkmcnt(0)
450; VI-NEXT:    v_mov_b32_e32 v4, s6
451; VI-NEXT:    v_mov_b32_e32 v5, s7
452; VI-NEXT:    v_mov_b32_e32 v6, s4
453; VI-NEXT:    v_mov_b32_e32 v7, s5
454; VI-NEXT:    flat_load_dwordx2 v[6:7], v[6:7]
455; VI-NEXT:    flat_load_dwordx2 v[4:5], v[4:5]
456; VI-NEXT:    v_mov_b32_e32 v2, s0
457; VI-NEXT:    v_mov_b32_e32 v3, s1
458; VI-NEXT:    v_mov_b32_e32 v0, s2
459; VI-NEXT:    v_mov_b32_e32 v1, s3
460; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
461; VI-NEXT:    v_add_u32_e32 v9, vcc, v7, v5
462; VI-NEXT:    v_add_u32_e32 v8, vcc, v6, v4
463; VI-NEXT:    v_cmp_gt_i32_e64 s[0:1], 0, v5
464; VI-NEXT:    v_cmp_lt_i32_e64 s[4:5], v9, v7
465; VI-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v4
466; VI-NEXT:    v_cmp_lt_i32_e64 s[2:3], v8, v6
467; VI-NEXT:    s_xor_b64 s[0:1], s[0:1], s[4:5]
468; VI-NEXT:    flat_store_dwordx2 v[2:3], v[8:9]
469; VI-NEXT:    v_cndmask_b32_e64 v3, 0, 1, s[0:1]
470; VI-NEXT:    s_xor_b64 s[0:1], vcc, s[2:3]
471; VI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
472; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
473; VI-NEXT:    s_endpgm
474;
475; GFX9-LABEL: v_saddo_v2i32:
476; GFX9:       ; %bb.0:
477; GFX9-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
478; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
479; GFX9-NEXT:    v_mov_b32_e32 v4, s6
480; GFX9-NEXT:    v_mov_b32_e32 v5, s7
481; GFX9-NEXT:    v_mov_b32_e32 v6, s4
482; GFX9-NEXT:    v_mov_b32_e32 v7, s5
483; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[6:7], off
484; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[4:5], off
485; GFX9-NEXT:    v_mov_b32_e32 v2, s0
486; GFX9-NEXT:    v_mov_b32_e32 v3, s1
487; GFX9-NEXT:    v_mov_b32_e32 v0, s2
488; GFX9-NEXT:    v_mov_b32_e32 v1, s3
489; GFX9-NEXT:    s_waitcnt vmcnt(0)
490; GFX9-NEXT:    v_add_u32_e32 v9, v7, v5
491; GFX9-NEXT:    v_add_u32_e32 v8, v6, v4
492; GFX9-NEXT:    v_cmp_gt_i32_e64 s[0:1], 0, v5
493; GFX9-NEXT:    v_cmp_lt_i32_e64 s[4:5], v9, v7
494; GFX9-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v4
495; GFX9-NEXT:    v_cmp_lt_i32_e64 s[2:3], v8, v6
496; GFX9-NEXT:    s_xor_b64 s[0:1], s[0:1], s[4:5]
497; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[8:9], off
498; GFX9-NEXT:    v_cndmask_b32_e64 v3, 0, 1, s[0:1]
499; GFX9-NEXT:    s_xor_b64 s[0:1], vcc, s[2:3]
500; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
501; GFX9-NEXT:    global_store_dwordx2 v[0:1], v[2:3], off
502; GFX9-NEXT:    s_endpgm
503  %a = load <2 x i32>, <2 x i32> addrspace(1)* %aptr, align 4
504  %b = load <2 x i32>, <2 x i32> addrspace(1)* %bptr, align 4
505  %sadd = call { <2 x i32>, <2 x i1> } @llvm.sadd.with.overflow.v2i32(<2 x i32> %a, <2 x i32> %b) nounwind
506  %val = extractvalue { <2 x i32>, <2 x i1> } %sadd, 0
507  %carry = extractvalue { <2 x i32>, <2 x i1> } %sadd, 1
508  store <2 x i32> %val, <2 x i32> addrspace(1)* %out, align 4
509  %carry.ext = zext <2 x i1> %carry to <2 x i32>
510  store <2 x i32> %carry.ext, <2 x i32> addrspace(1)* %carryout
511  ret void
512}
513