1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=SSE2
3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX2
4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX512F
5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=AVX512BW
6
7@a = global [1024 x i8] zeroinitializer, align 16
8@b = global [1024 x i8] zeroinitializer, align 16
9
10define i32 @sad_16i8() nounwind {
11; SSE2-LABEL: sad_16i8:
12; SSE2:       # BB#0: # %entry
13; SSE2-NEXT:    pxor %xmm0, %xmm0
14; SSE2-NEXT:    movq $-1024, %rax # imm = 0xFC00
15; SSE2-NEXT:    pxor %xmm1, %xmm1
16; SSE2-NEXT:    .p2align 4, 0x90
17; SSE2-NEXT:  .LBB0_1: # %vector.body
18; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
19; SSE2-NEXT:    movdqu a+1024(%rax), %xmm2
20; SSE2-NEXT:    movdqu b+1024(%rax), %xmm3
21; SSE2-NEXT:    psadbw %xmm2, %xmm3
22; SSE2-NEXT:    paddd %xmm3, %xmm1
23; SSE2-NEXT:    addq $4, %rax
24; SSE2-NEXT:    jne .LBB0_1
25; SSE2-NEXT:  # BB#2: # %middle.block
26; SSE2-NEXT:    paddd %xmm0, %xmm1
27; SSE2-NEXT:    paddd %xmm0, %xmm0
28; SSE2-NEXT:    paddd %xmm1, %xmm0
29; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
30; SSE2-NEXT:    paddd %xmm0, %xmm1
31; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
32; SSE2-NEXT:    paddd %xmm1, %xmm0
33; SSE2-NEXT:    movd %xmm0, %eax
34; SSE2-NEXT:    retq
35;
36; AVX2-LABEL: sad_16i8:
37; AVX2:       # BB#0: # %entry
38; AVX2-NEXT:    vpxor %ymm0, %ymm0, %ymm0
39; AVX2-NEXT:    movq $-1024, %rax # imm = 0xFC00
40; AVX2-NEXT:    vpxor %ymm1, %ymm1, %ymm1
41; AVX2-NEXT:    .p2align 4, 0x90
42; AVX2-NEXT:  .LBB0_1: # %vector.body
43; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
44; AVX2-NEXT:    vmovdqu a+1024(%rax), %xmm2
45; AVX2-NEXT:    vpsadbw b+1024(%rax), %xmm2, %xmm2
46; AVX2-NEXT:    vpaddd %xmm1, %xmm2, %xmm2
47; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
48; AVX2-NEXT:    addq $4, %rax
49; AVX2-NEXT:    jne .LBB0_1
50; AVX2-NEXT:  # BB#2: # %middle.block
51; AVX2-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
52; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
53; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
54; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
55; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
56; AVX2-NEXT:    vphaddd %ymm0, %ymm0, %ymm0
57; AVX2-NEXT:    vmovd %xmm0, %eax
58; AVX2-NEXT:    vzeroupper
59; AVX2-NEXT:    retq
60;
61; AVX512F-LABEL: sad_16i8:
62; AVX512F:       # BB#0: # %entry
63; AVX512F-NEXT:    vpxord %zmm0, %zmm0, %zmm0
64; AVX512F-NEXT:    movq $-1024, %rax # imm = 0xFC00
65; AVX512F-NEXT:    .p2align 4, 0x90
66; AVX512F-NEXT:  .LBB0_1: # %vector.body
67; AVX512F-NEXT:    # =>This Inner Loop Header: Depth=1
68; AVX512F-NEXT:    vmovdqu a+1024(%rax), %xmm1
69; AVX512F-NEXT:    vpsadbw b+1024(%rax), %xmm1, %xmm1
70; AVX512F-NEXT:    vpaddd %xmm0, %xmm1, %xmm1
71; AVX512F-NEXT:    vinserti32x4 $0, %xmm1, %zmm0, %zmm0
72; AVX512F-NEXT:    addq $4, %rax
73; AVX512F-NEXT:    jne .LBB0_1
74; AVX512F-NEXT:  # BB#2: # %middle.block
75; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm0[4,5,6,7,0,1,0,1]
76; AVX512F-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
77; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm0[2,3,0,1,0,1,0,1]
78; AVX512F-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
79; AVX512F-NEXT:    vpshufd {{.*#+}} zmm1 = zmm0[2,3,2,3,6,7,6,7,10,11,10,11,14,15,14,15]
80; AVX512F-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
81; AVX512F-NEXT:    vpshufd {{.*#+}} zmm1 = zmm0[1,1,2,3,5,5,6,7,9,9,10,11,13,13,14,15]
82; AVX512F-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
83; AVX512F-NEXT:    vmovd %xmm0, %eax
84; AVX512F-NEXT:    retq
85;
86; AVX512BW-LABEL: sad_16i8:
87; AVX512BW:       # BB#0: # %entry
88; AVX512BW-NEXT:    vpxord %zmm0, %zmm0, %zmm0
89; AVX512BW-NEXT:    movq $-1024, %rax # imm = 0xFC00
90; AVX512BW-NEXT:    .p2align 4, 0x90
91; AVX512BW-NEXT:  .LBB0_1: # %vector.body
92; AVX512BW-NEXT:    # =>This Inner Loop Header: Depth=1
93; AVX512BW-NEXT:    vmovdqu a+1024(%rax), %xmm1
94; AVX512BW-NEXT:    vpsadbw b+1024(%rax), %xmm1, %xmm1
95; AVX512BW-NEXT:    vpaddd %xmm0, %xmm1, %xmm1
96; AVX512BW-NEXT:    vinserti32x4 $0, %xmm1, %zmm0, %zmm0
97; AVX512BW-NEXT:    addq $4, %rax
98; AVX512BW-NEXT:    jne .LBB0_1
99; AVX512BW-NEXT:  # BB#2: # %middle.block
100; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm0[4,5,6,7,0,1,0,1]
101; AVX512BW-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
102; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm0[2,3,0,1,0,1,0,1]
103; AVX512BW-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
104; AVX512BW-NEXT:    vpshufd {{.*#+}} zmm1 = zmm0[2,3,2,3,6,7,6,7,10,11,10,11,14,15,14,15]
105; AVX512BW-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
106; AVX512BW-NEXT:    vpshufd {{.*#+}} zmm1 = zmm0[1,1,2,3,5,5,6,7,9,9,10,11,13,13,14,15]
107; AVX512BW-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
108; AVX512BW-NEXT:    vmovd %xmm0, %eax
109; AVX512BW-NEXT:    retq
110entry:
111  br label %vector.body
112
113vector.body:
114  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
115  %vec.phi = phi <16 x i32> [ zeroinitializer, %entry ], [ %10, %vector.body ]
116  %0 = getelementptr inbounds [1024 x i8], [1024 x i8]* @a, i64 0, i64 %index
117  %1 = bitcast i8* %0 to <16 x i8>*
118  %wide.load = load <16 x i8>, <16 x i8>* %1, align 4
119  %2 = zext <16 x i8> %wide.load to <16 x i32>
120  %3 = getelementptr inbounds [1024 x i8], [1024 x i8]* @b, i64 0, i64 %index
121  %4 = bitcast i8* %3 to <16 x i8>*
122  %wide.load1 = load <16 x i8>, <16 x i8>* %4, align 4
123  %5 = zext <16 x i8> %wide.load1 to <16 x i32>
124  %6 = sub nsw <16 x i32> %2, %5
125  %7 = icmp sgt <16 x i32> %6, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
126  %8 = sub nsw <16 x i32> zeroinitializer, %6
127  %9 = select <16 x i1> %7, <16 x i32> %6, <16 x i32> %8
128  %10 = add nsw <16 x i32> %9, %vec.phi
129  %index.next = add i64 %index, 4
130  %11 = icmp eq i64 %index.next, 1024
131  br i1 %11, label %middle.block, label %vector.body
132
133middle.block:
134  %.lcssa = phi <16 x i32> [ %10, %vector.body ]
135  %rdx.shuf = shufflevector <16 x i32> %.lcssa, <16 x i32> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
136  %bin.rdx = add <16 x i32> %.lcssa, %rdx.shuf
137  %rdx.shuf2 = shufflevector <16 x i32> %bin.rdx, <16 x i32> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
138  %bin.rdx2 = add <16 x i32> %bin.rdx, %rdx.shuf2
139  %rdx.shuf3 = shufflevector <16 x i32> %bin.rdx2, <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
140  %bin.rdx3 = add <16 x i32> %bin.rdx2, %rdx.shuf3
141  %rdx.shuf4 = shufflevector <16 x i32> %bin.rdx3, <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
142  %bin.rdx4 = add <16 x i32> %bin.rdx3, %rdx.shuf4
143  %12 = extractelement <16 x i32> %bin.rdx4, i32 0
144  ret i32 %12
145}
146
147define i32 @sad_32i8() nounwind {
148; SSE2-LABEL: sad_32i8:
149; SSE2:       # BB#0: # %entry
150; SSE2-NEXT:    pxor %xmm12, %xmm12
151; SSE2-NEXT:    movq $-1024, %rax # imm = 0xFC00
152; SSE2-NEXT:    pxor %xmm0, %xmm0
153; SSE2-NEXT:    pxor %xmm4, %xmm4
154; SSE2-NEXT:    pxor %xmm2, %xmm2
155; SSE2-NEXT:    pxor %xmm1, %xmm1
156; SSE2-NEXT:    pxor %xmm13, %xmm13
157; SSE2-NEXT:    pxor %xmm15, %xmm15
158; SSE2-NEXT:    pxor %xmm5, %xmm5
159; SSE2-NEXT:    pxor %xmm14, %xmm14
160; SSE2-NEXT:    .p2align 4, 0x90
161; SSE2-NEXT:  .LBB1_1: # %vector.body
162; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
163; SSE2-NEXT:    movdqa %xmm5, -{{[0-9]+}}(%rsp) # 16-byte Spill
164; SSE2-NEXT:    movdqa %xmm2, -{{[0-9]+}}(%rsp) # 16-byte Spill
165; SSE2-NEXT:    movdqa %xmm1, -{{[0-9]+}}(%rsp) # 16-byte Spill
166; SSE2-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp) # 16-byte Spill
167; SSE2-NEXT:    movdqa a+1040(%rax), %xmm1
168; SSE2-NEXT:    movdqa a+1024(%rax), %xmm3
169; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm3[2,3,0,1]
170; SSE2-NEXT:    punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm12[0],xmm8[1],xmm12[1],xmm8[2],xmm12[2],xmm8[3],xmm12[3],xmm8[4],xmm12[4],xmm8[5],xmm12[5],xmm8[6],xmm12[6],xmm8[7],xmm12[7]
171; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[2,3,0,1]
172; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm12[0],xmm7[1],xmm12[1],xmm7[2],xmm12[2],xmm7[3],xmm12[3],xmm7[4],xmm12[4],xmm7[5],xmm12[5],xmm7[6],xmm12[6],xmm7[7],xmm12[7]
173; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm12[0],xmm3[1],xmm12[1],xmm3[2],xmm12[2],xmm3[3],xmm12[3],xmm3[4],xmm12[4],xmm3[5],xmm12[5],xmm3[6],xmm12[6],xmm3[7],xmm12[7]
174; SSE2-NEXT:    movdqa %xmm3, %xmm6
175; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm12[4],xmm3[5],xmm12[5],xmm3[6],xmm12[6],xmm3[7],xmm12[7]
176; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm12[0],xmm1[1],xmm12[1],xmm1[2],xmm12[2],xmm1[3],xmm12[3],xmm1[4],xmm12[4],xmm1[5],xmm12[5],xmm1[6],xmm12[6],xmm1[7],xmm12[7]
177; SSE2-NEXT:    movdqa %xmm1, %xmm0
178; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm12[0],xmm0[1],xmm12[1],xmm0[2],xmm12[2],xmm0[3],xmm12[3]
179; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm12[4],xmm1[5],xmm12[5],xmm1[6],xmm12[6],xmm1[7],xmm12[7]
180; SSE2-NEXT:    movdqa b+1040(%rax), %xmm2
181; SSE2-NEXT:    movdqa b+1024(%rax), %xmm5
182; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm2[2,3,0,1]
183; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm12[0],xmm2[1],xmm12[1],xmm2[2],xmm12[2],xmm2[3],xmm12[3],xmm2[4],xmm12[4],xmm2[5],xmm12[5],xmm2[6],xmm12[6],xmm2[7],xmm12[7]
184; SSE2-NEXT:    movdqa %xmm2, %xmm10
185; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm12[4],xmm2[5],xmm12[5],xmm2[6],xmm12[6],xmm2[7],xmm12[7]
186; SSE2-NEXT:    psubd %xmm2, %xmm1
187; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm5[2,3,0,1]
188; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm12[0],xmm5[1],xmm12[1],xmm5[2],xmm12[2],xmm5[3],xmm12[3],xmm5[4],xmm12[4],xmm5[5],xmm12[5],xmm5[6],xmm12[6],xmm5[7],xmm12[7]
189; SSE2-NEXT:    punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm12[0],xmm10[1],xmm12[1],xmm10[2],xmm12[2],xmm10[3],xmm12[3]
190; SSE2-NEXT:    psubd %xmm10, %xmm0
191; SSE2-NEXT:    movdqa %xmm5, %xmm2
192; SSE2-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm12[4],xmm5[5],xmm12[5],xmm5[6],xmm12[6],xmm5[7],xmm12[7]
193; SSE2-NEXT:    psubd %xmm5, %xmm3
194; SSE2-NEXT:    movdqa %xmm7, %xmm5
195; SSE2-NEXT:    punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm12[4],xmm7[5],xmm12[5],xmm7[6],xmm12[6],xmm7[7],xmm12[7]
196; SSE2-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm12[0],xmm6[1],xmm12[1],xmm6[2],xmm12[2],xmm6[3],xmm12[3]
197; SSE2-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm12[0],xmm9[1],xmm12[1],xmm9[2],xmm12[2],xmm9[3],xmm12[3],xmm9[4],xmm12[4],xmm9[5],xmm12[5],xmm9[6],xmm12[6],xmm9[7],xmm12[7]
198; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm12[0],xmm2[1],xmm12[1],xmm2[2],xmm12[2],xmm2[3],xmm12[3]
199; SSE2-NEXT:    psubd %xmm2, %xmm6
200; SSE2-NEXT:    movdqa %xmm4, %xmm10
201; SSE2-NEXT:    movdqa %xmm9, %xmm4
202; SSE2-NEXT:    punpckhwd {{.*#+}} xmm9 = xmm9[4],xmm12[4],xmm9[5],xmm12[5],xmm9[6],xmm12[6],xmm9[7],xmm12[7]
203; SSE2-NEXT:    psubd %xmm9, %xmm7
204; SSE2-NEXT:    movdqa %xmm8, %xmm2
205; SSE2-NEXT:    punpckhwd {{.*#+}} xmm8 = xmm8[4],xmm12[4],xmm8[5],xmm12[5],xmm8[6],xmm12[6],xmm8[7],xmm12[7]
206; SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm12[0],xmm5[1],xmm12[1],xmm5[2],xmm12[2],xmm5[3],xmm12[3]
207; SSE2-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3],xmm11[4],xmm12[4],xmm11[5],xmm12[5],xmm11[6],xmm12[6],xmm11[7],xmm12[7]
208; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm12[0],xmm4[1],xmm12[1],xmm4[2],xmm12[2],xmm4[3],xmm12[3]
209; SSE2-NEXT:    psubd %xmm4, %xmm5
210; SSE2-NEXT:    movdqa %xmm11, %xmm4
211; SSE2-NEXT:    punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm12[4],xmm11[5],xmm12[5],xmm11[6],xmm12[6],xmm11[7],xmm12[7]
212; SSE2-NEXT:    psubd %xmm11, %xmm8
213; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm12[0],xmm2[1],xmm12[1],xmm2[2],xmm12[2],xmm2[3],xmm12[3]
214; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm12[0],xmm4[1],xmm12[1],xmm4[2],xmm12[2],xmm4[3],xmm12[3]
215; SSE2-NEXT:    psubd %xmm4, %xmm2
216; SSE2-NEXT:    movdqa %xmm2, %xmm4
217; SSE2-NEXT:    psrad $31, %xmm4
218; SSE2-NEXT:    paddd %xmm4, %xmm2
219; SSE2-NEXT:    pxor %xmm4, %xmm2
220; SSE2-NEXT:    movdqa %xmm8, %xmm4
221; SSE2-NEXT:    psrad $31, %xmm4
222; SSE2-NEXT:    paddd %xmm4, %xmm8
223; SSE2-NEXT:    pxor %xmm4, %xmm8
224; SSE2-NEXT:    movdqa %xmm5, %xmm4
225; SSE2-NEXT:    psrad $31, %xmm4
226; SSE2-NEXT:    paddd %xmm4, %xmm5
227; SSE2-NEXT:    pxor %xmm4, %xmm5
228; SSE2-NEXT:    movdqa %xmm7, %xmm4
229; SSE2-NEXT:    psrad $31, %xmm4
230; SSE2-NEXT:    paddd %xmm4, %xmm7
231; SSE2-NEXT:    pxor %xmm4, %xmm7
232; SSE2-NEXT:    movdqa %xmm6, %xmm4
233; SSE2-NEXT:    psrad $31, %xmm4
234; SSE2-NEXT:    paddd %xmm4, %xmm6
235; SSE2-NEXT:    pxor %xmm4, %xmm6
236; SSE2-NEXT:    movdqa %xmm3, %xmm4
237; SSE2-NEXT:    psrad $31, %xmm4
238; SSE2-NEXT:    paddd %xmm4, %xmm3
239; SSE2-NEXT:    pxor %xmm4, %xmm3
240; SSE2-NEXT:    movdqa %xmm0, %xmm4
241; SSE2-NEXT:    psrad $31, %xmm4
242; SSE2-NEXT:    paddd %xmm4, %xmm0
243; SSE2-NEXT:    pxor %xmm4, %xmm0
244; SSE2-NEXT:    movdqa %xmm1, %xmm4
245; SSE2-NEXT:    psrad $31, %xmm4
246; SSE2-NEXT:    paddd %xmm4, %xmm1
247; SSE2-NEXT:    pxor %xmm4, %xmm1
248; SSE2-NEXT:    movdqa %xmm10, %xmm4
249; SSE2-NEXT:    paddd %xmm1, %xmm15
250; SSE2-NEXT:    movdqa -{{[0-9]+}}(%rsp), %xmm1 # 16-byte Reload
251; SSE2-NEXT:    paddd %xmm0, %xmm13
252; SSE2-NEXT:    movdqa -{{[0-9]+}}(%rsp), %xmm0 # 16-byte Reload
253; SSE2-NEXT:    paddd %xmm3, %xmm4
254; SSE2-NEXT:    paddd %xmm6, %xmm0
255; SSE2-NEXT:    paddd %xmm7, %xmm14
256; SSE2-NEXT:    movdqa -{{[0-9]+}}(%rsp), %xmm3 # 16-byte Reload
257; SSE2-NEXT:    paddd %xmm5, %xmm3
258; SSE2-NEXT:    movdqa %xmm3, -{{[0-9]+}}(%rsp) # 16-byte Spill
259; SSE2-NEXT:    movdqa -{{[0-9]+}}(%rsp), %xmm5 # 16-byte Reload
260; SSE2-NEXT:    paddd %xmm8, %xmm1
261; SSE2-NEXT:    movdqa -{{[0-9]+}}(%rsp), %xmm3 # 16-byte Reload
262; SSE2-NEXT:    paddd %xmm2, %xmm3
263; SSE2-NEXT:    movdqa %xmm3, -{{[0-9]+}}(%rsp) # 16-byte Spill
264; SSE2-NEXT:    movdqa -{{[0-9]+}}(%rsp), %xmm2 # 16-byte Reload
265; SSE2-NEXT:    addq $4, %rax
266; SSE2-NEXT:    jne .LBB1_1
267; SSE2-NEXT:  # BB#2: # %middle.block
268; SSE2-NEXT:    paddd %xmm15, %xmm4
269; SSE2-NEXT:    paddd %xmm14, %xmm1
270; SSE2-NEXT:    paddd %xmm13, %xmm0
271; SSE2-NEXT:    paddd %xmm5, %xmm2
272; SSE2-NEXT:    paddd %xmm4, %xmm1
273; SSE2-NEXT:    paddd %xmm2, %xmm1
274; SSE2-NEXT:    paddd %xmm0, %xmm1
275; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
276; SSE2-NEXT:    paddd %xmm1, %xmm0
277; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
278; SSE2-NEXT:    paddd %xmm0, %xmm1
279; SSE2-NEXT:    movd %xmm1, %eax
280; SSE2-NEXT:    retq
281;
282; AVX2-LABEL: sad_32i8:
283; AVX2:       # BB#0: # %entry
284; AVX2-NEXT:    vpxor %ymm0, %ymm0, %ymm0
285; AVX2-NEXT:    movq $-1024, %rax # imm = 0xFC00
286; AVX2-NEXT:    vpxor %ymm1, %ymm1, %ymm1
287; AVX2-NEXT:    .p2align 4, 0x90
288; AVX2-NEXT:  .LBB1_1: # %vector.body
289; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
290; AVX2-NEXT:    vmovdqa a+1024(%rax), %ymm2
291; AVX2-NEXT:    vpsadbw b+1024(%rax), %ymm2, %ymm2
292; AVX2-NEXT:    vpaddd %ymm1, %ymm2, %ymm1
293; AVX2-NEXT:    addq $4, %rax
294; AVX2-NEXT:    jne .LBB1_1
295; AVX2-NEXT:  # BB#2: # %middle.block
296; AVX2-NEXT:    vpaddd %ymm0, %ymm1, %ymm1
297; AVX2-NEXT:    vpaddd %ymm0, %ymm0, %ymm0
298; AVX2-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
299; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
300; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
301; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
302; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
303; AVX2-NEXT:    vphaddd %ymm0, %ymm0, %ymm0
304; AVX2-NEXT:    vmovd %xmm0, %eax
305; AVX2-NEXT:    vzeroupper
306; AVX2-NEXT:    retq
307;
308; AVX512F-LABEL: sad_32i8:
309; AVX512F:       # BB#0: # %entry
310; AVX512F-NEXT:    vpxord %zmm0, %zmm0, %zmm0
311; AVX512F-NEXT:    movq $-1024, %rax # imm = 0xFC00
312; AVX512F-NEXT:    vpxord %zmm1, %zmm1, %zmm1
313; AVX512F-NEXT:    .p2align 4, 0x90
314; AVX512F-NEXT:  .LBB1_1: # %vector.body
315; AVX512F-NEXT:    # =>This Inner Loop Header: Depth=1
316; AVX512F-NEXT:    vmovdqa a+1024(%rax), %ymm2
317; AVX512F-NEXT:    vpsadbw b+1024(%rax), %ymm2, %ymm2
318; AVX512F-NEXT:    vpaddd %ymm1, %ymm2, %ymm2
319; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm1, %zmm1
320; AVX512F-NEXT:    addq $4, %rax
321; AVX512F-NEXT:    jne .LBB1_1
322; AVX512F-NEXT:  # BB#2: # %middle.block
323; AVX512F-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
324; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm0[4,5,6,7,0,1,0,1]
325; AVX512F-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
326; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm0[2,3,0,1,0,1,0,1]
327; AVX512F-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
328; AVX512F-NEXT:    vpshufd {{.*#+}} zmm1 = zmm0[2,3,2,3,6,7,6,7,10,11,10,11,14,15,14,15]
329; AVX512F-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
330; AVX512F-NEXT:    vpshufd {{.*#+}} zmm1 = zmm0[1,1,2,3,5,5,6,7,9,9,10,11,13,13,14,15]
331; AVX512F-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
332; AVX512F-NEXT:    vmovd %xmm0, %eax
333; AVX512F-NEXT:    retq
334;
335; AVX512BW-LABEL: sad_32i8:
336; AVX512BW:       # BB#0: # %entry
337; AVX512BW-NEXT:    vpxord %zmm0, %zmm0, %zmm0
338; AVX512BW-NEXT:    movq $-1024, %rax # imm = 0xFC00
339; AVX512BW-NEXT:    vpxord %zmm1, %zmm1, %zmm1
340; AVX512BW-NEXT:    .p2align 4, 0x90
341; AVX512BW-NEXT:  .LBB1_1: # %vector.body
342; AVX512BW-NEXT:    # =>This Inner Loop Header: Depth=1
343; AVX512BW-NEXT:    vmovdqa a+1024(%rax), %ymm2
344; AVX512BW-NEXT:    vpsadbw b+1024(%rax), %ymm2, %ymm2
345; AVX512BW-NEXT:    vpaddd %ymm1, %ymm2, %ymm2
346; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm1, %zmm1
347; AVX512BW-NEXT:    addq $4, %rax
348; AVX512BW-NEXT:    jne .LBB1_1
349; AVX512BW-NEXT:  # BB#2: # %middle.block
350; AVX512BW-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
351; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm0[4,5,6,7,0,1,0,1]
352; AVX512BW-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
353; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm0[2,3,0,1,0,1,0,1]
354; AVX512BW-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
355; AVX512BW-NEXT:    vpshufd {{.*#+}} zmm1 = zmm0[2,3,2,3,6,7,6,7,10,11,10,11,14,15,14,15]
356; AVX512BW-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
357; AVX512BW-NEXT:    vpshufd {{.*#+}} zmm1 = zmm0[1,1,2,3,5,5,6,7,9,9,10,11,13,13,14,15]
358; AVX512BW-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
359; AVX512BW-NEXT:    vmovd %xmm0, %eax
360; AVX512BW-NEXT:    retq
361entry:
362  br label %vector.body
363
364vector.body:
365  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
366  %vec.phi = phi <32 x i32> [ zeroinitializer, %entry ], [ %10, %vector.body ]
367  %0 = getelementptr inbounds [1024 x i8], [1024 x i8]* @a, i64 0, i64 %index
368  %1 = bitcast i8* %0 to <32 x i8>*
369  %wide.load = load <32 x i8>, <32 x i8>* %1, align 32
370  %2 = zext <32 x i8> %wide.load to <32 x i32>
371  %3 = getelementptr inbounds [1024 x i8], [1024 x i8]* @b, i64 0, i64 %index
372  %4 = bitcast i8* %3 to <32 x i8>*
373  %wide.load1 = load <32 x i8>, <32 x i8>* %4, align 32
374  %5 = zext <32 x i8> %wide.load1 to <32 x i32>
375  %6 = sub nsw <32 x i32> %2, %5
376  %7 = icmp sgt <32 x i32> %6, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
377  %8 = sub nsw <32 x i32> zeroinitializer, %6
378  %9 = select <32 x i1> %7, <32 x i32> %6, <32 x i32> %8
379  %10 = add nsw <32 x i32> %9, %vec.phi
380  %index.next = add i64 %index, 4
381  %11 = icmp eq i64 %index.next, 1024
382  br i1 %11, label %middle.block, label %vector.body
383
384middle.block:
385  %.lcssa = phi <32 x i32> [ %10, %vector.body ]
386  %rdx.shuf = shufflevector <32 x i32> %.lcssa, <32 x i32> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
387  %bin.rdx = add <32 x i32> %.lcssa, %rdx.shuf
388  %rdx.shuf2 = shufflevector <32 x i32> %bin.rdx, <32 x i32> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
389  %bin.rdx2 = add <32 x i32> %bin.rdx, %rdx.shuf2
390  %rdx.shuf3 = shufflevector <32 x i32> %bin.rdx2, <32 x i32> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
391  %bin.rdx3 = add <32 x i32> %bin.rdx2, %rdx.shuf3
392  %rdx.shuf4 = shufflevector <32 x i32> %bin.rdx3, <32 x i32> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
393  %bin.rdx4 = add <32 x i32> %bin.rdx3, %rdx.shuf4
394  %rdx.shuf5 = shufflevector <32 x i32> %bin.rdx4, <32 x i32> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
395  %bin.rdx5 = add <32 x i32> %bin.rdx4, %rdx.shuf5
396  %12 = extractelement <32 x i32> %bin.rdx5, i32 0
397  ret i32 %12
398}
399
400define i32 @sad_avx64i8() nounwind {
401; SSE2-LABEL: sad_avx64i8:
402; SSE2:       # BB#0: # %entry
403; SSE2-NEXT:    subq $216, %rsp
404; SSE2-NEXT:    pxor %xmm6, %xmm6
405; SSE2-NEXT:    movq $-1024, %rax # imm = 0xFC00
406; SSE2-NEXT:    pxor %xmm0, %xmm0
407; SSE2-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp) # 16-byte Spill
408; SSE2-NEXT:    pxor %xmm0, %xmm0
409; SSE2-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp) # 16-byte Spill
410; SSE2-NEXT:    pxor %xmm3, %xmm3
411; SSE2-NEXT:    pxor %xmm2, %xmm2
412; SSE2-NEXT:    pxor %xmm13, %xmm13
413; SSE2-NEXT:    pxor %xmm0, %xmm0
414; SSE2-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp) # 16-byte Spill
415; SSE2-NEXT:    pxor %xmm10, %xmm10
416; SSE2-NEXT:    pxor %xmm12, %xmm12
417; SSE2-NEXT:    pxor %xmm0, %xmm0
418; SSE2-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp) # 16-byte Spill
419; SSE2-NEXT:    pxor %xmm1, %xmm1
420; SSE2-NEXT:    pxor %xmm15, %xmm15
421; SSE2-NEXT:    pxor %xmm11, %xmm11
422; SSE2-NEXT:    pxor %xmm8, %xmm8
423; SSE2-NEXT:    pxor %xmm4, %xmm4
424; SSE2-NEXT:    pxor %xmm5, %xmm5
425; SSE2-NEXT:    pxor %xmm0, %xmm0
426; SSE2-NEXT:    .p2align 4, 0x90
427; SSE2-NEXT:  .LBB2_1: # %vector.body
428; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
429; SSE2-NEXT:    movdqa %xmm5, {{[0-9]+}}(%rsp) # 16-byte Spill
430; SSE2-NEXT:    movdqa %xmm15, {{[0-9]+}}(%rsp) # 16-byte Spill
431; SSE2-NEXT:    movdqa %xmm8, -{{[0-9]+}}(%rsp) # 16-byte Spill
432; SSE2-NEXT:    movdqa %xmm0, (%rsp) # 16-byte Spill
433; SSE2-NEXT:    movdqa %xmm11, {{[0-9]+}}(%rsp) # 16-byte Spill
434; SSE2-NEXT:    movdqa %xmm4, {{[0-9]+}}(%rsp) # 16-byte Spill
435; SSE2-NEXT:    movdqa %xmm1, {{[0-9]+}}(%rsp) # 16-byte Spill
436; SSE2-NEXT:    movdqa %xmm2, {{[0-9]+}}(%rsp) # 16-byte Spill
437; SSE2-NEXT:    movdqa %xmm13, {{[0-9]+}}(%rsp) # 16-byte Spill
438; SSE2-NEXT:    movdqa %xmm3, {{[0-9]+}}(%rsp) # 16-byte Spill
439; SSE2-NEXT:    movdqa %xmm10, {{[0-9]+}}(%rsp) # 16-byte Spill
440; SSE2-NEXT:    movdqa %xmm12, {{[0-9]+}}(%rsp) # 16-byte Spill
441; SSE2-NEXT:    movdqa a+1040(%rax), %xmm13
442; SSE2-NEXT:    movdqa a+1024(%rax), %xmm12
443; SSE2-NEXT:    movdqa a+1056(%rax), %xmm10
444; SSE2-NEXT:    movdqa a+1072(%rax), %xmm8
445; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm10[2,3,0,1]
446; SSE2-NEXT:    movdqa %xmm0, {{[0-9]+}}(%rsp) # 16-byte Spill
447; SSE2-NEXT:    punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm6[0],xmm10[1],xmm6[1],xmm10[2],xmm6[2],xmm10[3],xmm6[3],xmm10[4],xmm6[4],xmm10[5],xmm6[5],xmm10[6],xmm6[6],xmm10[7],xmm6[7]
448; SSE2-NEXT:    movdqa %xmm10, -{{[0-9]+}}(%rsp) # 16-byte Spill
449; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm12[2,3,0,1]
450; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3],xmm3[4],xmm6[4],xmm3[5],xmm6[5],xmm3[6],xmm6[6],xmm3[7],xmm6[7]
451; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm13[2,3,0,1]
452; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3],xmm4[4],xmm6[4],xmm4[5],xmm6[5],xmm4[6],xmm6[6],xmm4[7],xmm6[7]
453; SSE2-NEXT:    movdqa %xmm4, %xmm1
454; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm6[4],xmm4[5],xmm6[5],xmm4[6],xmm6[6],xmm4[7],xmm6[7]
455; SSE2-NEXT:    punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm6[0],xmm10[1],xmm6[1],xmm10[2],xmm6[2],xmm10[3],xmm6[3]
456; SSE2-NEXT:    punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm6[0],xmm12[1],xmm6[1],xmm12[2],xmm6[2],xmm12[3],xmm6[3],xmm12[4],xmm6[4],xmm12[5],xmm6[5],xmm12[6],xmm6[6],xmm12[7],xmm6[7]
457; SSE2-NEXT:    movdqa %xmm12, %xmm0
458; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3]
459; SSE2-NEXT:    movdqa %xmm0, %xmm15
460; SSE2-NEXT:    punpckhwd {{.*#+}} xmm12 = xmm12[4],xmm6[4],xmm12[5],xmm6[5],xmm12[6],xmm6[6],xmm12[7],xmm6[7]
461; SSE2-NEXT:    punpcklbw {{.*#+}} xmm13 = xmm13[0],xmm6[0],xmm13[1],xmm6[1],xmm13[2],xmm6[2],xmm13[3],xmm6[3],xmm13[4],xmm6[4],xmm13[5],xmm6[5],xmm13[6],xmm6[6],xmm13[7],xmm6[7]
462; SSE2-NEXT:    movdqa %xmm13, %xmm0
463; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3]
464; SSE2-NEXT:    punpckhwd {{.*#+}} xmm13 = xmm13[4],xmm6[4],xmm13[5],xmm6[5],xmm13[6],xmm6[6],xmm13[7],xmm6[7]
465; SSE2-NEXT:    movdqa b+1040(%rax), %xmm7
466; SSE2-NEXT:    movdqa b+1024(%rax), %xmm11
467; SSE2-NEXT:    movdqa b+1056(%rax), %xmm9
468; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[2,3,0,1]
469; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3],xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
470; SSE2-NEXT:    movdqa %xmm7, %xmm2
471; SSE2-NEXT:    punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
472; SSE2-NEXT:    psubd %xmm7, %xmm13
473; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm11[2,3,0,1]
474; SSE2-NEXT:    punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm6[0],xmm11[1],xmm6[1],xmm11[2],xmm6[2],xmm11[3],xmm6[3],xmm11[4],xmm6[4],xmm11[5],xmm6[5],xmm11[6],xmm6[6],xmm11[7],xmm6[7]
475; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3]
476; SSE2-NEXT:    psubd %xmm2, %xmm0
477; SSE2-NEXT:    movdqa %xmm0, {{[0-9]+}}(%rsp) # 16-byte Spill
478; SSE2-NEXT:    movdqa %xmm11, %xmm2
479; SSE2-NEXT:    punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm6[4],xmm11[5],xmm6[5],xmm11[6],xmm6[6],xmm11[7],xmm6[7]
480; SSE2-NEXT:    psubd %xmm11, %xmm12
481; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm9[2,3,0,1]
482; SSE2-NEXT:    punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm6[0],xmm9[1],xmm6[1],xmm9[2],xmm6[2],xmm9[3],xmm6[3],xmm9[4],xmm6[4],xmm9[5],xmm6[5],xmm9[6],xmm6[6],xmm9[7],xmm6[7]
483; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3]
484; SSE2-NEXT:    psubd %xmm2, %xmm15
485; SSE2-NEXT:    movdqa %xmm15, -{{[0-9]+}}(%rsp) # 16-byte Spill
486; SSE2-NEXT:    movdqa %xmm9, %xmm2
487; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3],xmm5[4],xmm6[4],xmm5[5],xmm6[5],xmm5[6],xmm6[6],xmm5[7],xmm6[7]
488; SSE2-NEXT:    punpcklwd {{.*#+}} xmm9 = xmm9[0],xmm6[0],xmm9[1],xmm6[1],xmm9[2],xmm6[2],xmm9[3],xmm6[3]
489; SSE2-NEXT:    psubd %xmm9, %xmm10
490; SSE2-NEXT:    movdqa %xmm5, %xmm0
491; SSE2-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm6[4],xmm5[5],xmm6[5],xmm5[6],xmm6[6],xmm5[7],xmm6[7]
492; SSE2-NEXT:    psubd %xmm5, %xmm4
493; SSE2-NEXT:    movdqa %xmm3, %xmm15
494; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm6[4],xmm3[5],xmm6[5],xmm3[6],xmm6[6],xmm3[7],xmm6[7]
495; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1],xmm1[2],xmm6[2],xmm1[3],xmm6[3]
496; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3],xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
497; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3]
498; SSE2-NEXT:    psubd %xmm0, %xmm1
499; SSE2-NEXT:    movdqa %xmm1, -{{[0-9]+}}(%rsp) # 16-byte Spill
500; SSE2-NEXT:    movdqa %xmm7, %xmm0
501; SSE2-NEXT:    punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
502; SSE2-NEXT:    psubd %xmm7, %xmm3
503; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm8[2,3,0,1]
504; SSE2-NEXT:    punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3],xmm8[4],xmm6[4],xmm8[5],xmm6[5],xmm8[6],xmm6[6],xmm8[7],xmm6[7]
505; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm1 # 16-byte Reload
506; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1],xmm1[2],xmm6[2],xmm1[3],xmm6[3],xmm1[4],xmm6[4],xmm1[5],xmm6[5],xmm1[6],xmm6[6],xmm1[7],xmm6[7]
507; SSE2-NEXT:    punpcklwd {{.*#+}} xmm15 = xmm15[0],xmm6[0],xmm15[1],xmm6[1],xmm15[2],xmm6[2],xmm15[3],xmm6[3]
508; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3]
509; SSE2-NEXT:    psubd %xmm0, %xmm15
510; SSE2-NEXT:    movdqa %xmm1, %xmm11
511; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1],xmm1[2],xmm6[2],xmm1[3],xmm6[3]
512; SSE2-NEXT:    movdqa -{{[0-9]+}}(%rsp), %xmm0 # 16-byte Reload
513; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm6[4],xmm0[5],xmm6[5],xmm0[6],xmm6[6],xmm0[7],xmm6[7]
514; SSE2-NEXT:    punpcklbw {{.*#+}} xmm14 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3],xmm14[4],xmm6[4],xmm14[5],xmm6[5],xmm14[6],xmm6[6],xmm14[7],xmm6[7]
515; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7]
516; SSE2-NEXT:    psubd %xmm2, %xmm0
517; SSE2-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp) # 16-byte Spill
518; SSE2-NEXT:    movdqa %xmm14, %xmm0
519; SSE2-NEXT:    punpcklwd {{.*#+}} xmm14 = xmm14[0],xmm6[0],xmm14[1],xmm6[1],xmm14[2],xmm6[2],xmm14[3],xmm6[3]
520; SSE2-NEXT:    psubd %xmm14, %xmm1
521; SSE2-NEXT:    movdqa %xmm1, %xmm14
522; SSE2-NEXT:    movdqa %xmm8, %xmm9
523; SSE2-NEXT:    punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]
524; SSE2-NEXT:    punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm6[4],xmm11[5],xmm6[5],xmm11[6],xmm6[6],xmm11[7],xmm6[7]
525; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm6[4],xmm0[5],xmm6[5],xmm0[6],xmm6[6],xmm0[7],xmm6[7]
526; SSE2-NEXT:    psubd %xmm0, %xmm11
527; SSE2-NEXT:    movdqa b+1072(%rax), %xmm0
528; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
529; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3],xmm0[4],xmm6[4],xmm0[5],xmm6[5],xmm0[6],xmm6[6],xmm0[7],xmm6[7]
530; SSE2-NEXT:    movdqa %xmm0, %xmm5
531; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3]
532; SSE2-NEXT:    psubd %xmm0, %xmm8
533; SSE2-NEXT:    punpckhwd {{.*#+}} xmm9 = xmm9[4],xmm6[4],xmm9[5],xmm6[5],xmm9[6],xmm6[6],xmm9[7],xmm6[7]
534; SSE2-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm6[4],xmm5[5],xmm6[5],xmm5[6],xmm6[6],xmm5[7],xmm6[7]
535; SSE2-NEXT:    psubd %xmm5, %xmm9
536; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3],xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
537; SSE2-NEXT:    movdqa %xmm7, %xmm0
538; SSE2-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
539; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3],xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7]
540; SSE2-NEXT:    movdqa %xmm2, %xmm5
541; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3]
542; SSE2-NEXT:    psubd %xmm2, %xmm7
543; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm6[4],xmm0[5],xmm6[5],xmm0[6],xmm6[6],xmm0[7],xmm6[7]
544; SSE2-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm6[4],xmm5[5],xmm6[5],xmm5[6],xmm6[6],xmm5[7],xmm6[7]
545; SSE2-NEXT:    psubd %xmm5, %xmm0
546; SSE2-NEXT:    movdqa %xmm0, %xmm2
547; SSE2-NEXT:    psrad $31, %xmm2
548; SSE2-NEXT:    paddd %xmm2, %xmm0
549; SSE2-NEXT:    pxor %xmm2, %xmm0
550; SSE2-NEXT:    movdqa %xmm7, %xmm2
551; SSE2-NEXT:    psrad $31, %xmm2
552; SSE2-NEXT:    paddd %xmm2, %xmm7
553; SSE2-NEXT:    pxor %xmm2, %xmm7
554; SSE2-NEXT:    movdqa %xmm9, %xmm2
555; SSE2-NEXT:    psrad $31, %xmm2
556; SSE2-NEXT:    paddd %xmm2, %xmm9
557; SSE2-NEXT:    pxor %xmm2, %xmm9
558; SSE2-NEXT:    movdqa %xmm8, %xmm2
559; SSE2-NEXT:    psrad $31, %xmm2
560; SSE2-NEXT:    paddd %xmm2, %xmm8
561; SSE2-NEXT:    pxor %xmm2, %xmm8
562; SSE2-NEXT:    movdqa %xmm11, %xmm2
563; SSE2-NEXT:    psrad $31, %xmm2
564; SSE2-NEXT:    paddd %xmm2, %xmm11
565; SSE2-NEXT:    pxor %xmm2, %xmm11
566; SSE2-NEXT:    movdqa %xmm14, %xmm2
567; SSE2-NEXT:    psrad $31, %xmm2
568; SSE2-NEXT:    paddd %xmm2, %xmm14
569; SSE2-NEXT:    pxor %xmm2, %xmm14
570; SSE2-NEXT:    movdqa -{{[0-9]+}}(%rsp), %xmm1 # 16-byte Reload
571; SSE2-NEXT:    movdqa %xmm1, %xmm2
572; SSE2-NEXT:    psrad $31, %xmm2
573; SSE2-NEXT:    paddd %xmm2, %xmm1
574; SSE2-NEXT:    pxor %xmm2, %xmm1
575; SSE2-NEXT:    movdqa %xmm1, -{{[0-9]+}}(%rsp) # 16-byte Spill
576; SSE2-NEXT:    movdqa %xmm15, %xmm2
577; SSE2-NEXT:    psrad $31, %xmm2
578; SSE2-NEXT:    paddd %xmm2, %xmm15
579; SSE2-NEXT:    pxor %xmm2, %xmm15
580; SSE2-NEXT:    movdqa %xmm3, %xmm2
581; SSE2-NEXT:    psrad $31, %xmm2
582; SSE2-NEXT:    paddd %xmm2, %xmm3
583; SSE2-NEXT:    pxor %xmm2, %xmm3
584; SSE2-NEXT:    movdqa -{{[0-9]+}}(%rsp), %xmm1 # 16-byte Reload
585; SSE2-NEXT:    movdqa %xmm1, %xmm2
586; SSE2-NEXT:    psrad $31, %xmm2
587; SSE2-NEXT:    paddd %xmm2, %xmm1
588; SSE2-NEXT:    pxor %xmm2, %xmm1
589; SSE2-NEXT:    movdqa %xmm1, -{{[0-9]+}}(%rsp) # 16-byte Spill
590; SSE2-NEXT:    movdqa %xmm4, %xmm2
591; SSE2-NEXT:    psrad $31, %xmm2
592; SSE2-NEXT:    paddd %xmm2, %xmm4
593; SSE2-NEXT:    pxor %xmm2, %xmm4
594; SSE2-NEXT:    movdqa %xmm10, %xmm2
595; SSE2-NEXT:    psrad $31, %xmm2
596; SSE2-NEXT:    paddd %xmm2, %xmm10
597; SSE2-NEXT:    pxor %xmm2, %xmm10
598; SSE2-NEXT:    movdqa -{{[0-9]+}}(%rsp), %xmm1 # 16-byte Reload
599; SSE2-NEXT:    movdqa %xmm1, %xmm2
600; SSE2-NEXT:    psrad $31, %xmm2
601; SSE2-NEXT:    paddd %xmm2, %xmm1
602; SSE2-NEXT:    pxor %xmm2, %xmm1
603; SSE2-NEXT:    movdqa %xmm1, -{{[0-9]+}}(%rsp) # 16-byte Spill
604; SSE2-NEXT:    movdqa %xmm12, %xmm2
605; SSE2-NEXT:    psrad $31, %xmm2
606; SSE2-NEXT:    paddd %xmm2, %xmm12
607; SSE2-NEXT:    pxor %xmm2, %xmm12
608; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm1 # 16-byte Reload
609; SSE2-NEXT:    movdqa %xmm1, %xmm2
610; SSE2-NEXT:    psrad $31, %xmm2
611; SSE2-NEXT:    paddd %xmm2, %xmm1
612; SSE2-NEXT:    pxor %xmm2, %xmm1
613; SSE2-NEXT:    movdqa %xmm13, %xmm2
614; SSE2-NEXT:    psrad $31, %xmm2
615; SSE2-NEXT:    paddd %xmm2, %xmm13
616; SSE2-NEXT:    pxor %xmm2, %xmm13
617; SSE2-NEXT:    movdqa -{{[0-9]+}}(%rsp), %xmm5 # 16-byte Reload
618; SSE2-NEXT:    paddd %xmm13, %xmm5
619; SSE2-NEXT:    movdqa %xmm5, -{{[0-9]+}}(%rsp) # 16-byte Spill
620; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm13 # 16-byte Reload
621; SSE2-NEXT:    paddd %xmm1, %xmm13
622; SSE2-NEXT:    movdqa -{{[0-9]+}}(%rsp), %xmm5 # 16-byte Reload
623; SSE2-NEXT:    paddd %xmm12, %xmm5
624; SSE2-NEXT:    movdqa %xmm5, -{{[0-9]+}}(%rsp) # 16-byte Spill
625; SSE2-NEXT:    movdqa -{{[0-9]+}}(%rsp), %xmm1 # 16-byte Reload
626; SSE2-NEXT:    paddd -{{[0-9]+}}(%rsp), %xmm1 # 16-byte Folded Reload
627; SSE2-NEXT:    movdqa %xmm1, -{{[0-9]+}}(%rsp) # 16-byte Spill
628; SSE2-NEXT:    movdqa -{{[0-9]+}}(%rsp), %xmm1 # 16-byte Reload
629; SSE2-NEXT:    paddd %xmm10, %xmm1
630; SSE2-NEXT:    movdqa %xmm1, -{{[0-9]+}}(%rsp) # 16-byte Spill
631; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm1 # 16-byte Reload
632; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm2 # 16-byte Reload
633; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm5 # 16-byte Reload
634; SSE2-NEXT:    paddd %xmm4, %xmm5
635; SSE2-NEXT:    movdqa %xmm5, {{[0-9]+}}(%rsp) # 16-byte Spill
636; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm10 # 16-byte Reload
637; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm12 # 16-byte Reload
638; SSE2-NEXT:    paddd -{{[0-9]+}}(%rsp), %xmm10 # 16-byte Folded Reload
639; SSE2-NEXT:    paddd %xmm3, %xmm2
640; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm3 # 16-byte Reload
641; SSE2-NEXT:    paddd %xmm15, %xmm3
642; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm15 # 16-byte Reload
643; SSE2-NEXT:    paddd -{{[0-9]+}}(%rsp), %xmm1 # 16-byte Folded Reload
644; SSE2-NEXT:    paddd %xmm14, %xmm15
645; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm4 # 16-byte Reload
646; SSE2-NEXT:    paddd %xmm11, %xmm4
647; SSE2-NEXT:    movdqa %xmm4, {{[0-9]+}}(%rsp) # 16-byte Spill
648; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm11 # 16-byte Reload
649; SSE2-NEXT:    movdqa -{{[0-9]+}}(%rsp), %xmm4 # 16-byte Reload
650; SSE2-NEXT:    paddd %xmm8, %xmm4
651; SSE2-NEXT:    movdqa %xmm4, -{{[0-9]+}}(%rsp) # 16-byte Spill
652; SSE2-NEXT:    movdqa -{{[0-9]+}}(%rsp), %xmm8 # 16-byte Reload
653; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm4 # 16-byte Reload
654; SSE2-NEXT:    paddd %xmm9, %xmm4
655; SSE2-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm5 # 16-byte Reload
656; SSE2-NEXT:    paddd %xmm7, %xmm5
657; SSE2-NEXT:    movdqa (%rsp), %xmm7 # 16-byte Reload
658; SSE2-NEXT:    paddd %xmm0, %xmm7
659; SSE2-NEXT:    movdqa %xmm7, (%rsp) # 16-byte Spill
660; SSE2-NEXT:    movdqa (%rsp), %xmm0 # 16-byte Reload
661; SSE2-NEXT:    addq $4, %rax
662; SSE2-NEXT:    jne .LBB2_1
663; SSE2-NEXT:  # BB#2: # %middle.block
664; SSE2-NEXT:    paddd %xmm15, %xmm3
665; SSE2-NEXT:    paddd %xmm5, %xmm10
666; SSE2-NEXT:    movdqa -{{[0-9]+}}(%rsp), %xmm6 # 16-byte Reload
667; SSE2-NEXT:    paddd -{{[0-9]+}}(%rsp), %xmm6 # 16-byte Folded Reload
668; SSE2-NEXT:    paddd %xmm8, %xmm13
669; SSE2-NEXT:    paddd %xmm11, %xmm2
670; SSE2-NEXT:    paddd %xmm0, %xmm12
671; SSE2-NEXT:    movdqa -{{[0-9]+}}(%rsp), %xmm5 # 16-byte Reload
672; SSE2-NEXT:    paddd %xmm1, %xmm5
673; SSE2-NEXT:    movdqa -{{[0-9]+}}(%rsp), %xmm0 # 16-byte Reload
674; SSE2-NEXT:    paddd %xmm4, %xmm0
675; SSE2-NEXT:    paddd %xmm2, %xmm12
676; SSE2-NEXT:    paddd %xmm3, %xmm10
677; SSE2-NEXT:    paddd %xmm13, %xmm10
678; SSE2-NEXT:    paddd %xmm0, %xmm12
679; SSE2-NEXT:    paddd %xmm5, %xmm12
680; SSE2-NEXT:    paddd %xmm10, %xmm12
681; SSE2-NEXT:    paddd %xmm6, %xmm12
682; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm12[2,3,0,1]
683; SSE2-NEXT:    paddd %xmm12, %xmm0
684; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
685; SSE2-NEXT:    paddd %xmm0, %xmm1
686; SSE2-NEXT:    movd %xmm1, %eax
687; SSE2-NEXT:    addq $216, %rsp
688; SSE2-NEXT:    retq
689;
690; AVX2-LABEL: sad_avx64i8:
691; AVX2:       # BB#0: # %entry
692; AVX2-NEXT:    vpxor %ymm0, %ymm0, %ymm0
693; AVX2-NEXT:    movq $-1024, %rax # imm = 0xFC00
694; AVX2-NEXT:    vpxor %ymm2, %ymm2, %ymm2
695; AVX2-NEXT:    vpxor %ymm1, %ymm1, %ymm1
696; AVX2-NEXT:    vpxor %ymm3, %ymm3, %ymm3
697; AVX2-NEXT:    vpxor %ymm4, %ymm4, %ymm4
698; AVX2-NEXT:    vpxor %ymm6, %ymm6, %ymm6
699; AVX2-NEXT:    vpxor %ymm5, %ymm5, %ymm5
700; AVX2-NEXT:    vpxor %ymm7, %ymm7, %ymm7
701; AVX2-NEXT:    .p2align 4, 0x90
702; AVX2-NEXT:  .LBB2_1: # %vector.body
703; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
704; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
705; AVX2-NEXT:    vmovdqu %ymm8, -{{[0-9]+}}(%rsp) # 32-byte Spill
706; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm9 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
707; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm10 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
708; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm11 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
709; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm12 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
710; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm13 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
711; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm14 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
712; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm15 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
713; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
714; AVX2-NEXT:    vpsubd %ymm8, %ymm15, %ymm8
715; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm15 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
716; AVX2-NEXT:    vpsubd %ymm15, %ymm14, %ymm14
717; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm15 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
718; AVX2-NEXT:    vpsubd %ymm15, %ymm13, %ymm13
719; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm15 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
720; AVX2-NEXT:    vpsubd %ymm15, %ymm12, %ymm12
721; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm15 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
722; AVX2-NEXT:    vpsubd %ymm15, %ymm11, %ymm11
723; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm15 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
724; AVX2-NEXT:    vpsubd %ymm15, %ymm10, %ymm10
725; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm15 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
726; AVX2-NEXT:    vpsubd %ymm15, %ymm9, %ymm9
727; AVX2-NEXT:    vmovdqu %ymm9, -{{[0-9]+}}(%rsp) # 32-byte Spill
728; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm15 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
729; AVX2-NEXT:    vmovdqu -{{[0-9]+}}(%rsp), %ymm9 # 32-byte Reload
730; AVX2-NEXT:    vpsubd %ymm15, %ymm9, %ymm15
731; AVX2-NEXT:    vpabsd %ymm8, %ymm8
732; AVX2-NEXT:    vpaddd %ymm3, %ymm8, %ymm3
733; AVX2-NEXT:    vpabsd %ymm14, %ymm8
734; AVX2-NEXT:    vpaddd %ymm1, %ymm8, %ymm1
735; AVX2-NEXT:    vpabsd %ymm13, %ymm8
736; AVX2-NEXT:    vpaddd %ymm2, %ymm8, %ymm2
737; AVX2-NEXT:    vpabsd %ymm12, %ymm8
738; AVX2-NEXT:    vpaddd %ymm0, %ymm8, %ymm0
739; AVX2-NEXT:    vpabsd %ymm11, %ymm8
740; AVX2-NEXT:    vpaddd %ymm4, %ymm8, %ymm4
741; AVX2-NEXT:    vpabsd %ymm10, %ymm8
742; AVX2-NEXT:    vpaddd %ymm6, %ymm8, %ymm6
743; AVX2-NEXT:    vpabsd -{{[0-9]+}}(%rsp), %ymm8 # 32-byte Folded Reload
744; AVX2-NEXT:    vpaddd %ymm5, %ymm8, %ymm5
745; AVX2-NEXT:    vpabsd %ymm15, %ymm8
746; AVX2-NEXT:    vpaddd %ymm7, %ymm8, %ymm7
747; AVX2-NEXT:    addq $4, %rax
748; AVX2-NEXT:    jne .LBB2_1
749; AVX2-NEXT:  # BB#2: # %middle.block
750; AVX2-NEXT:    vpaddd %ymm6, %ymm2, %ymm2
751; AVX2-NEXT:    vpaddd %ymm7, %ymm3, %ymm3
752; AVX2-NEXT:    vpaddd %ymm4, %ymm0, %ymm0
753; AVX2-NEXT:    vpaddd %ymm5, %ymm1, %ymm1
754; AVX2-NEXT:    vpaddd %ymm3, %ymm2, %ymm2
755; AVX2-NEXT:    vpaddd %ymm2, %ymm1, %ymm1
756; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
757; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
758; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
759; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
760; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
761; AVX2-NEXT:    vphaddd %ymm0, %ymm0, %ymm0
762; AVX2-NEXT:    vmovd %xmm0, %eax
763; AVX2-NEXT:    vzeroupper
764; AVX2-NEXT:    retq
765;
766; AVX512F-LABEL: sad_avx64i8:
767; AVX512F:       # BB#0: # %entry
768; AVX512F-NEXT:    vpxord %zmm0, %zmm0, %zmm0
769; AVX512F-NEXT:    movq $-1024, %rax # imm = 0xFC00
770; AVX512F-NEXT:    vpxord %zmm1, %zmm1, %zmm1
771; AVX512F-NEXT:    vpxord %zmm2, %zmm2, %zmm2
772; AVX512F-NEXT:    vpxord %zmm3, %zmm3, %zmm3
773; AVX512F-NEXT:    .p2align 4, 0x90
774; AVX512F-NEXT:  .LBB2_1: # %vector.body
775; AVX512F-NEXT:    # =>This Inner Loop Header: Depth=1
776; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
777; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm5 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
778; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm6 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
779; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm7 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
780; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
781; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm9 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
782; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm10 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
783; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm11 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
784; AVX512F-NEXT:    vpsubd %zmm11, %zmm7, %zmm7
785; AVX512F-NEXT:    vpsubd %zmm10, %zmm6, %zmm6
786; AVX512F-NEXT:    vpsubd %zmm9, %zmm5, %zmm5
787; AVX512F-NEXT:    vpsubd %zmm8, %zmm4, %zmm4
788; AVX512F-NEXT:    vpabsd %zmm4, %zmm4
789; AVX512F-NEXT:    vpabsd %zmm5, %zmm5
790; AVX512F-NEXT:    vpabsd %zmm6, %zmm6
791; AVX512F-NEXT:    vpabsd %zmm7, %zmm7
792; AVX512F-NEXT:    vpaddd %zmm3, %zmm7, %zmm3
793; AVX512F-NEXT:    vpaddd %zmm2, %zmm6, %zmm2
794; AVX512F-NEXT:    vpaddd %zmm1, %zmm5, %zmm1
795; AVX512F-NEXT:    vpaddd %zmm0, %zmm4, %zmm0
796; AVX512F-NEXT:    addq $4, %rax
797; AVX512F-NEXT:    jne .LBB2_1
798; AVX512F-NEXT:  # BB#2: # %middle.block
799; AVX512F-NEXT:    vpaddd %zmm2, %zmm0, %zmm0
800; AVX512F-NEXT:    vpaddd %zmm3, %zmm1, %zmm1
801; AVX512F-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
802; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm0[4,5,6,7,0,1,0,1]
803; AVX512F-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
804; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm0[2,3,0,1,0,1,0,1]
805; AVX512F-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
806; AVX512F-NEXT:    vpshufd {{.*#+}} zmm1 = zmm0[2,3,2,3,6,7,6,7,10,11,10,11,14,15,14,15]
807; AVX512F-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
808; AVX512F-NEXT:    vpshufd {{.*#+}} zmm1 = zmm0[1,1,2,3,5,5,6,7,9,9,10,11,13,13,14,15]
809; AVX512F-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
810; AVX512F-NEXT:    vmovd %xmm0, %eax
811; AVX512F-NEXT:    retq
812;
813; AVX512BW-LABEL: sad_avx64i8:
814; AVX512BW:       # BB#0: # %entry
815; AVX512BW-NEXT:    vpxord %zmm0, %zmm0, %zmm0
816; AVX512BW-NEXT:    movq $-1024, %rax # imm = 0xFC00
817; AVX512BW-NEXT:    vpxord %zmm1, %zmm1, %zmm1
818; AVX512BW-NEXT:    .p2align 4, 0x90
819; AVX512BW-NEXT:  .LBB2_1: # %vector.body
820; AVX512BW-NEXT:    # =>This Inner Loop Header: Depth=1
821; AVX512BW-NEXT:    vmovdqu8 a+1024(%rax), %zmm2
822; AVX512BW-NEXT:    vpsadbw b+1024(%rax), %zmm2, %zmm2
823; AVX512BW-NEXT:    vpaddd %zmm1, %zmm2, %zmm1
824; AVX512BW-NEXT:    addq $4, %rax
825; AVX512BW-NEXT:    jne .LBB2_1
826; AVX512BW-NEXT:  # BB#2: # %middle.block
827; AVX512BW-NEXT:    vpaddd %zmm0, %zmm1, %zmm1
828; AVX512BW-NEXT:    vpaddd %zmm0, %zmm0, %zmm0
829; AVX512BW-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
830; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm0[4,5,6,7,0,1,0,1]
831; AVX512BW-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
832; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm0[2,3,0,1,0,1,0,1]
833; AVX512BW-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
834; AVX512BW-NEXT:    vpshufd {{.*#+}} zmm1 = zmm0[2,3,2,3,6,7,6,7,10,11,10,11,14,15,14,15]
835; AVX512BW-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
836; AVX512BW-NEXT:    vpshufd {{.*#+}} zmm1 = zmm0[1,1,2,3,5,5,6,7,9,9,10,11,13,13,14,15]
837; AVX512BW-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
838; AVX512BW-NEXT:    vmovd %xmm0, %eax
839; AVX512BW-NEXT:    retq
840entry:
841  br label %vector.body
842
843vector.body:
844  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
845  %vec.phi = phi <64 x i32> [ zeroinitializer, %entry ], [ %10, %vector.body ]
846  %0 = getelementptr inbounds [1024 x i8], [1024 x i8]* @a, i64 0, i64 %index
847  %1 = bitcast i8* %0 to <64 x i8>*
848  %wide.load = load <64 x i8>, <64 x i8>* %1, align 64
849  %2 = zext <64 x i8> %wide.load to <64 x i32>
850  %3 = getelementptr inbounds [1024 x i8], [1024 x i8]* @b, i64 0, i64 %index
851  %4 = bitcast i8* %3 to <64 x i8>*
852  %wide.load1 = load <64 x i8>, <64 x i8>* %4, align 64
853  %5 = zext <64 x i8> %wide.load1 to <64 x i32>
854  %6 = sub nsw <64 x i32> %2, %5
855  %7 = icmp sgt <64 x i32> %6, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
856  %8 = sub nsw <64 x i32> zeroinitializer, %6
857  %9 = select <64 x i1> %7, <64 x i32> %6, <64 x i32> %8
858  %10 = add nsw <64 x i32> %9, %vec.phi
859  %index.next = add i64 %index, 4
860  %11 = icmp eq i64 %index.next, 1024
861  br i1 %11, label %middle.block, label %vector.body
862
863middle.block:
864  %.lcssa = phi <64 x i32> [ %10, %vector.body ]
865  %rdx.shuf = shufflevector <64 x i32> %.lcssa, <64 x i32> undef, <64 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
866  %bin.rdx = add <64 x i32> %.lcssa, %rdx.shuf
867  %rdx.shuf2 = shufflevector <64 x i32> %bin.rdx, <64 x i32> undef, <64 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
868  %bin.rdx2 = add <64 x i32> %bin.rdx, %rdx.shuf2
869  %rdx.shuf3 = shufflevector <64 x i32> %bin.rdx2, <64 x i32> undef, <64 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
870  %bin.rdx3 = add <64 x i32> %bin.rdx2, %rdx.shuf3
871  %rdx.shuf4 = shufflevector <64 x i32> %bin.rdx3, <64 x i32> undef, <64 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
872  %bin.rdx4 = add <64 x i32> %bin.rdx3, %rdx.shuf4
873  %rdx.shuf5 = shufflevector <64 x i32> %bin.rdx4, <64 x i32> undef, <64 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
874  %bin.rdx5 = add <64 x i32> %bin.rdx4, %rdx.shuf5
875  %rdx.shuf6 = shufflevector <64 x i32> %bin.rdx5, <64 x i32> undef, <64 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
876  %bin.rdx6 = add <64 x i32> %bin.rdx5, %rdx.shuf6
877  %12 = extractelement <64 x i32> %bin.rdx6, i32 0
878  ret i32 %12
879}
880
881define i32 @sad_2i8() nounwind {
882; SSE2-LABEL: sad_2i8:
883; SSE2:       # BB#0: # %entry
884; SSE2-NEXT:    pxor %xmm0, %xmm0
885; SSE2-NEXT:    movq $-1024, %rax # imm = 0xFC00
886; SSE2-NEXT:    movl $65535, %ecx # imm = 0xFFFF
887; SSE2-NEXT:    movd %ecx, %xmm1
888; SSE2-NEXT:    .p2align 4, 0x90
889; SSE2-NEXT:  .LBB3_1: # %vector.body
890; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
891; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
892; SSE2-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
893; SSE2-NEXT:    pand %xmm1, %xmm3
894; SSE2-NEXT:    pand %xmm1, %xmm2
895; SSE2-NEXT:    psadbw %xmm3, %xmm2
896; SSE2-NEXT:    paddq %xmm2, %xmm0
897; SSE2-NEXT:    addq $4, %rax
898; SSE2-NEXT:    jne .LBB3_1
899; SSE2-NEXT:  # BB#2: # %middle.block
900; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
901; SSE2-NEXT:    paddq %xmm0, %xmm1
902; SSE2-NEXT:    movd %xmm1, %eax
903; SSE2-NEXT:    retq
904;
905; AVX2-LABEL: sad_2i8:
906; AVX2:       # BB#0: # %entry
907; AVX2-NEXT:    vpxor %xmm0, %xmm0, %xmm0
908; AVX2-NEXT:    movq $-1024, %rax # imm = 0xFC00
909; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
910; AVX2-NEXT:    .p2align 4, 0x90
911; AVX2-NEXT:  .LBB3_1: # %vector.body
912; AVX2-NEXT:    # =>This Inner Loop Header: Depth=1
913; AVX2-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
914; AVX2-NEXT:    vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
915; AVX2-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0],xmm0[1,2,3,4,5,6,7]
916; AVX2-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm0[1,2,3,4,5,6,7]
917; AVX2-NEXT:    vpsadbw %xmm3, %xmm2, %xmm2
918; AVX2-NEXT:    vpaddq %xmm1, %xmm2, %xmm1
919; AVX2-NEXT:    addq $4, %rax
920; AVX2-NEXT:    jne .LBB3_1
921; AVX2-NEXT:  # BB#2: # %middle.block
922; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
923; AVX2-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
924; AVX2-NEXT:    vmovd %xmm0, %eax
925; AVX2-NEXT:    retq
926;
927; AVX512F-LABEL: sad_2i8:
928; AVX512F:       # BB#0: # %entry
929; AVX512F-NEXT:    vpxor %xmm0, %xmm0, %xmm0
930; AVX512F-NEXT:    movq $-1024, %rax # imm = 0xFC00
931; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
932; AVX512F-NEXT:    .p2align 4, 0x90
933; AVX512F-NEXT:  .LBB3_1: # %vector.body
934; AVX512F-NEXT:    # =>This Inner Loop Header: Depth=1
935; AVX512F-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
936; AVX512F-NEXT:    vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
937; AVX512F-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0],xmm0[1,2,3,4,5,6,7]
938; AVX512F-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm0[1,2,3,4,5,6,7]
939; AVX512F-NEXT:    vpsadbw %xmm3, %xmm2, %xmm2
940; AVX512F-NEXT:    vpaddq %xmm1, %xmm2, %xmm1
941; AVX512F-NEXT:    addq $4, %rax
942; AVX512F-NEXT:    jne .LBB3_1
943; AVX512F-NEXT:  # BB#2: # %middle.block
944; AVX512F-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
945; AVX512F-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
946; AVX512F-NEXT:    vmovd %xmm0, %eax
947; AVX512F-NEXT:    retq
948;
949; AVX512BW-LABEL: sad_2i8:
950; AVX512BW:       # BB#0: # %entry
951; AVX512BW-NEXT:    vpxor %xmm0, %xmm0, %xmm0
952; AVX512BW-NEXT:    movq $-1024, %rax # imm = 0xFC00
953; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
954; AVX512BW-NEXT:    .p2align 4, 0x90
955; AVX512BW-NEXT:  .LBB3_1: # %vector.body
956; AVX512BW-NEXT:    # =>This Inner Loop Header: Depth=1
957; AVX512BW-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
958; AVX512BW-NEXT:    vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
959; AVX512BW-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0],xmm0[1,2,3,4,5,6,7]
960; AVX512BW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm0[1,2,3,4,5,6,7]
961; AVX512BW-NEXT:    vpsadbw %xmm3, %xmm2, %xmm2
962; AVX512BW-NEXT:    vpaddq %xmm1, %xmm2, %xmm1
963; AVX512BW-NEXT:    addq $4, %rax
964; AVX512BW-NEXT:    jne .LBB3_1
965; AVX512BW-NEXT:  # BB#2: # %middle.block
966; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
967; AVX512BW-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
968; AVX512BW-NEXT:    vmovd %xmm0, %eax
969; AVX512BW-NEXT:    retq
970entry:
971  br label %vector.body
972
973vector.body:
974  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
975  %vec.phi = phi <2 x i32> [ zeroinitializer, %entry ], [ %10, %vector.body ]
976  %0 = getelementptr inbounds [1024 x i8], [1024 x i8]* @a, i64 0, i64 %index
977  %1 = bitcast i8* %0 to <2 x i8>*
978  %wide.load = load <2 x i8>, <2 x i8>* %1, align 4
979  %2 = zext <2 x i8> %wide.load to <2 x i32>
980  %3 = getelementptr inbounds [1024 x i8], [1024 x i8]* @b, i64 0, i64 %index
981  %4 = bitcast i8* %3 to <2 x i8>*
982  %wide.load1 = load <2 x i8>, <2 x i8>* %4, align 4
983  %5 = zext <2 x i8> %wide.load1 to <2 x i32>
984  %6 = sub nsw <2 x i32> %2, %5
985  %7 = icmp sgt <2 x i32> %6, <i32 -1, i32 -1>
986  %8 = sub nsw <2 x i32> zeroinitializer, %6
987  %9 = select <2 x i1> %7, <2 x i32> %6, <2 x i32> %8
988  %10 = add nsw <2 x i32> %9, %vec.phi
989  %index.next = add i64 %index, 4
990  %11 = icmp eq i64 %index.next, 1024
991  br i1 %11, label %middle.block, label %vector.body
992
993middle.block:
994  %.lcssa = phi <2 x i32> [ %10, %vector.body ]
995  %rdx.shuf = shufflevector <2 x i32> %.lcssa, <2 x i32> undef, <2 x i32> <i32 1, i32 undef>
996  %bin.rdx = add <2 x i32> %.lcssa, %rdx.shuf
997  %12 = extractelement <2 x i32> %bin.rdx, i32 0
998  ret i32 %12
999}
1000
1001define i32 @sad_nonloop_4i8(<4 x i8>* nocapture readonly %p, i64, <4 x i8>* nocapture readonly %q) local_unnamed_addr #0 {
1002; SSE2-LABEL: sad_nonloop_4i8:
1003; SSE2:       # BB#0:
1004; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
1005; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
1006; SSE2-NEXT:    psadbw %xmm0, %xmm1
1007; SSE2-NEXT:    movd %xmm1, %eax
1008; SSE2-NEXT:    retq
1009;
1010; AVX2-LABEL: sad_nonloop_4i8:
1011; AVX2:       # BB#0:
1012; AVX2-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
1013; AVX2-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
1014; AVX2-NEXT:    vpsadbw %xmm0, %xmm1, %xmm0
1015; AVX2-NEXT:    vmovd %xmm0, %eax
1016; AVX2-NEXT:    retq
1017;
1018; AVX512F-LABEL: sad_nonloop_4i8:
1019; AVX512F:       # BB#0:
1020; AVX512F-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
1021; AVX512F-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
1022; AVX512F-NEXT:    vpsadbw %xmm0, %xmm1, %xmm0
1023; AVX512F-NEXT:    vmovd %xmm0, %eax
1024; AVX512F-NEXT:    retq
1025;
1026; AVX512BW-LABEL: sad_nonloop_4i8:
1027; AVX512BW:       # BB#0:
1028; AVX512BW-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
1029; AVX512BW-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
1030; AVX512BW-NEXT:    vpsadbw %xmm0, %xmm1, %xmm0
1031; AVX512BW-NEXT:    vmovd %xmm0, %eax
1032; AVX512BW-NEXT:    retq
1033  %v1 = load <4 x i8>, <4 x i8>* %p, align 1
1034  %z1 = zext <4 x i8> %v1 to <4 x i32>
1035  %v2 = load <4 x i8>, <4 x i8>* %q, align 1
1036  %z2 = zext <4 x i8> %v2 to <4 x i32>
1037  %sub = sub nsw <4 x i32> %z1, %z2
1038  %isneg = icmp sgt <4 x i32> %sub, <i32 -1, i32 -1, i32 -1, i32 -1>
1039  %neg = sub nsw <4 x i32> zeroinitializer, %sub
1040  %abs = select <4 x i1> %isneg, <4 x i32> %sub, <4 x i32> %neg
1041  %h2 = shufflevector <4 x i32> %abs, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
1042  %sum2 = add <4 x i32> %abs, %h2
1043  %h3 = shufflevector <4 x i32> %sum2, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
1044  %sum3 = add <4 x i32> %sum2, %h3
1045  %sum = extractelement <4 x i32> %sum3, i32 0
1046  ret i32 %sum
1047}
1048
1049define i32 @sad_nonloop_8i8(<8 x i8>* nocapture readonly %p, i64, <8 x i8>* nocapture readonly %q) local_unnamed_addr #0 {
1050; SSE2-LABEL: sad_nonloop_8i8:
1051; SSE2:       # BB#0:
1052; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
1053; SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
1054; SSE2-NEXT:    psadbw %xmm0, %xmm1
1055; SSE2-NEXT:    movd %xmm1, %eax
1056; SSE2-NEXT:    retq
1057;
1058; AVX2-LABEL: sad_nonloop_8i8:
1059; AVX2:       # BB#0:
1060; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
1061; AVX2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
1062; AVX2-NEXT:    vpsadbw %xmm0, %xmm1, %xmm0
1063; AVX2-NEXT:    vmovd %xmm0, %eax
1064; AVX2-NEXT:    retq
1065;
1066; AVX512F-LABEL: sad_nonloop_8i8:
1067; AVX512F:       # BB#0:
1068; AVX512F-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
1069; AVX512F-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
1070; AVX512F-NEXT:    vpsadbw %xmm0, %xmm1, %xmm0
1071; AVX512F-NEXT:    vmovd %xmm0, %eax
1072; AVX512F-NEXT:    retq
1073;
1074; AVX512BW-LABEL: sad_nonloop_8i8:
1075; AVX512BW:       # BB#0:
1076; AVX512BW-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
1077; AVX512BW-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
1078; AVX512BW-NEXT:    vpsadbw %xmm0, %xmm1, %xmm0
1079; AVX512BW-NEXT:    vmovd %xmm0, %eax
1080; AVX512BW-NEXT:    retq
1081  %v1 = load <8 x i8>, <8 x i8>* %p, align 1
1082  %z1 = zext <8 x i8> %v1 to <8 x i32>
1083  %v2 = load <8 x i8>, <8 x i8>* %q, align 1
1084  %z2 = zext <8 x i8> %v2 to <8 x i32>
1085  %sub = sub nsw <8 x i32> %z1, %z2
1086  %isneg = icmp sgt <8 x i32> %sub, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
1087  %neg = sub nsw <8 x i32> zeroinitializer, %sub
1088  %abs = select <8 x i1> %isneg, <8 x i32> %sub, <8 x i32> %neg
1089  %h1 = shufflevector <8 x i32> %abs, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
1090  %sum1 = add <8 x i32> %abs, %h1
1091  %h2 = shufflevector <8 x i32> %sum1, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1092  %sum2 = add <8 x i32> %sum1, %h2
1093  %h3 = shufflevector <8 x i32> %sum2, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1094  %sum3 = add <8 x i32> %sum2, %h3
1095  %sum = extractelement <8 x i32> %sum3, i32 0
1096  ret i32 %sum
1097}
1098
1099define i32 @sad_nonloop_16i8(<16 x i8>* nocapture readonly %p, i64, <16 x i8>* nocapture readonly %q) local_unnamed_addr #0 {
1100; SSE2-LABEL: sad_nonloop_16i8:
1101; SSE2:       # BB#0:
1102; SSE2-NEXT:    movdqu (%rdi), %xmm0
1103; SSE2-NEXT:    movdqu (%rdx), %xmm1
1104; SSE2-NEXT:    psadbw %xmm0, %xmm1
1105; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
1106; SSE2-NEXT:    paddq %xmm1, %xmm0
1107; SSE2-NEXT:    movd %xmm0, %eax
1108; SSE2-NEXT:    retq
1109;
1110; AVX2-LABEL: sad_nonloop_16i8:
1111; AVX2:       # BB#0:
1112; AVX2-NEXT:    vmovdqu (%rdi), %xmm0
1113; AVX2-NEXT:    vpsadbw (%rdx), %xmm0, %xmm0
1114; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1115; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
1116; AVX2-NEXT:    vmovd %xmm0, %eax
1117; AVX2-NEXT:    retq
1118;
1119; AVX512F-LABEL: sad_nonloop_16i8:
1120; AVX512F:       # BB#0:
1121; AVX512F-NEXT:    vmovdqu (%rdi), %xmm0
1122; AVX512F-NEXT:    vpsadbw (%rdx), %xmm0, %xmm0
1123; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1124; AVX512F-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
1125; AVX512F-NEXT:    vmovd %xmm0, %eax
1126; AVX512F-NEXT:    retq
1127;
1128; AVX512BW-LABEL: sad_nonloop_16i8:
1129; AVX512BW:       # BB#0:
1130; AVX512BW-NEXT:    vmovdqu (%rdi), %xmm0
1131; AVX512BW-NEXT:    vpsadbw (%rdx), %xmm0, %xmm0
1132; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1133; AVX512BW-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
1134; AVX512BW-NEXT:    vmovd %xmm0, %eax
1135; AVX512BW-NEXT:    retq
1136  %v1 = load <16 x i8>, <16 x i8>* %p, align 1
1137  %z1 = zext <16 x i8> %v1 to <16 x i32>
1138  %v2 = load <16 x i8>, <16 x i8>* %q, align 1
1139  %z2 = zext <16 x i8> %v2 to <16 x i32>
1140  %sub = sub nsw <16 x i32> %z1, %z2
1141  %isneg = icmp sgt <16 x i32> %sub, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
1142  %neg = sub nsw <16 x i32> zeroinitializer, %sub
1143  %abs = select <16 x i1> %isneg, <16 x i32> %sub, <16 x i32> %neg
1144  %h0 = shufflevector <16 x i32> %abs, <16 x i32> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1145  %sum0 = add <16 x i32> %abs, %h0
1146  %h1 = shufflevector <16 x i32> %sum0, <16 x i32> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1147  %sum1 = add <16 x i32> %sum0, %h1
1148  %h2 = shufflevector <16 x i32> %sum1, <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1149  %sum2 = add <16 x i32> %sum1, %h2
1150  %h3 = shufflevector <16 x i32> %sum2, <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1151  %sum3 = add <16 x i32> %sum2, %h3
1152  %sum = extractelement <16 x i32> %sum3, i32 0
1153  ret i32 %sum
1154}
1155
1156define i32 @sad_nonloop_32i8(<32 x i8>* nocapture readonly %p, i64, <32 x i8>* nocapture readonly %q) local_unnamed_addr #0 {
1157; SSE2-LABEL: sad_nonloop_32i8:
1158; SSE2:       # BB#0:
1159; SSE2-NEXT:    movdqu (%rdi), %xmm12
1160; SSE2-NEXT:    movdqu 16(%rdi), %xmm2
1161; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm2[2,3,0,1]
1162; SSE2-NEXT:    pxor %xmm5, %xmm5
1163; SSE2-NEXT:    punpcklbw {{.*#+}} xmm13 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3],xmm13[4],xmm5[4],xmm13[5],xmm5[5],xmm13[6],xmm5[6],xmm13[7],xmm5[7]
1164; SSE2-NEXT:    movdqa %xmm13, %xmm9
1165; SSE2-NEXT:    punpckhwd {{.*#+}} xmm9 = xmm9[4],xmm5[4],xmm9[5],xmm5[5],xmm9[6],xmm5[6],xmm9[7],xmm5[7]
1166; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm12[2,3,0,1]
1167; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3],xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7]
1168; SSE2-NEXT:    movdqa %xmm1, %xmm3
1169; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
1170; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7]
1171; SSE2-NEXT:    movdqa %xmm2, %xmm10
1172; SSE2-NEXT:    punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm5[4],xmm10[5],xmm5[5],xmm10[6],xmm5[6],xmm10[7],xmm5[7]
1173; SSE2-NEXT:    punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm5[0],xmm12[1],xmm5[1],xmm12[2],xmm5[2],xmm12[3],xmm5[3],xmm12[4],xmm5[4],xmm12[5],xmm5[5],xmm12[6],xmm5[6],xmm12[7],xmm5[7]
1174; SSE2-NEXT:    movdqa %xmm12, %xmm11
1175; SSE2-NEXT:    punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm5[4],xmm11[5],xmm5[5],xmm11[6],xmm5[6],xmm11[7],xmm5[7]
1176; SSE2-NEXT:    punpcklwd {{.*#+}} xmm13 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3]
1177; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
1178; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]
1179; SSE2-NEXT:    punpcklwd {{.*#+}} xmm12 = xmm12[0],xmm5[0],xmm12[1],xmm5[1],xmm12[2],xmm5[2],xmm12[3],xmm5[3]
1180; SSE2-NEXT:    movdqu (%rdx), %xmm7
1181; SSE2-NEXT:    movdqu 16(%rdx), %xmm0
1182; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[2,3,0,1]
1183; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3],xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7]
1184; SSE2-NEXT:    movdqa %xmm6, %xmm4
1185; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]
1186; SSE2-NEXT:    movdqa %xmm4, -{{[0-9]+}}(%rsp) # 16-byte Spill
1187; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[2,3,0,1]
1188; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3],xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]
1189; SSE2-NEXT:    movdqa %xmm4, %xmm14
1190; SSE2-NEXT:    punpckhwd {{.*#+}} xmm14 = xmm14[4],xmm5[4],xmm14[5],xmm5[5],xmm14[6],xmm5[6],xmm14[7],xmm5[7]
1191; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3],xmm0[4],xmm5[4],xmm0[5],xmm5[5],xmm0[6],xmm5[6],xmm0[7],xmm5[7]
1192; SSE2-NEXT:    movdqa %xmm0, %xmm15
1193; SSE2-NEXT:    punpckhwd {{.*#+}} xmm15 = xmm15[4],xmm5[4],xmm15[5],xmm5[5],xmm15[6],xmm5[6],xmm15[7],xmm5[7]
1194; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3],xmm7[4],xmm5[4],xmm7[5],xmm5[5],xmm7[6],xmm5[6],xmm7[7],xmm5[7]
1195; SSE2-NEXT:    movdqa %xmm7, %xmm8
1196; SSE2-NEXT:    punpckhwd {{.*#+}} xmm8 = xmm8[4],xmm5[4],xmm8[5],xmm5[5],xmm8[6],xmm5[6],xmm8[7],xmm5[7]
1197; SSE2-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
1198; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
1199; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
1200; SSE2-NEXT:    punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]
1201; SSE2-NEXT:    psubd %xmm7, %xmm12
1202; SSE2-NEXT:    psubd %xmm0, %xmm2
1203; SSE2-NEXT:    psubd %xmm4, %xmm1
1204; SSE2-NEXT:    psubd %xmm6, %xmm13
1205; SSE2-NEXT:    psubd %xmm8, %xmm11
1206; SSE2-NEXT:    psubd %xmm15, %xmm10
1207; SSE2-NEXT:    psubd %xmm14, %xmm3
1208; SSE2-NEXT:    psubd -{{[0-9]+}}(%rsp), %xmm9 # 16-byte Folded Reload
1209; SSE2-NEXT:    movdqa %xmm9, %xmm0
1210; SSE2-NEXT:    psrad $31, %xmm0
1211; SSE2-NEXT:    paddd %xmm0, %xmm9
1212; SSE2-NEXT:    pxor %xmm0, %xmm9
1213; SSE2-NEXT:    movdqa %xmm3, %xmm0
1214; SSE2-NEXT:    psrad $31, %xmm0
1215; SSE2-NEXT:    paddd %xmm0, %xmm3
1216; SSE2-NEXT:    pxor %xmm0, %xmm3
1217; SSE2-NEXT:    movdqa %xmm10, %xmm0
1218; SSE2-NEXT:    psrad $31, %xmm0
1219; SSE2-NEXT:    paddd %xmm0, %xmm10
1220; SSE2-NEXT:    pxor %xmm0, %xmm10
1221; SSE2-NEXT:    movdqa %xmm11, %xmm0
1222; SSE2-NEXT:    psrad $31, %xmm0
1223; SSE2-NEXT:    paddd %xmm0, %xmm11
1224; SSE2-NEXT:    pxor %xmm0, %xmm11
1225; SSE2-NEXT:    movdqa %xmm13, %xmm0
1226; SSE2-NEXT:    psrad $31, %xmm0
1227; SSE2-NEXT:    paddd %xmm0, %xmm13
1228; SSE2-NEXT:    pxor %xmm0, %xmm13
1229; SSE2-NEXT:    movdqa %xmm1, %xmm0
1230; SSE2-NEXT:    psrad $31, %xmm0
1231; SSE2-NEXT:    paddd %xmm0, %xmm1
1232; SSE2-NEXT:    pxor %xmm0, %xmm1
1233; SSE2-NEXT:    movdqa %xmm2, %xmm0
1234; SSE2-NEXT:    psrad $31, %xmm0
1235; SSE2-NEXT:    paddd %xmm0, %xmm2
1236; SSE2-NEXT:    pxor %xmm0, %xmm2
1237; SSE2-NEXT:    movdqa %xmm12, %xmm0
1238; SSE2-NEXT:    psrad $31, %xmm0
1239; SSE2-NEXT:    paddd %xmm0, %xmm12
1240; SSE2-NEXT:    pxor %xmm0, %xmm12
1241; SSE2-NEXT:    paddd %xmm13, %xmm1
1242; SSE2-NEXT:    paddd %xmm9, %xmm3
1243; SSE2-NEXT:    paddd %xmm10, %xmm3
1244; SSE2-NEXT:    paddd %xmm11, %xmm3
1245; SSE2-NEXT:    paddd %xmm2, %xmm1
1246; SSE2-NEXT:    paddd %xmm3, %xmm1
1247; SSE2-NEXT:    paddd %xmm12, %xmm1
1248; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
1249; SSE2-NEXT:    paddd %xmm1, %xmm0
1250; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
1251; SSE2-NEXT:    paddd %xmm0, %xmm1
1252; SSE2-NEXT:    movd %xmm1, %eax
1253; SSE2-NEXT:    retq
1254;
1255; AVX2-LABEL: sad_nonloop_32i8:
1256; AVX2:       # BB#0:
1257; AVX2-NEXT:    vmovdqu (%rdi), %ymm0
1258; AVX2-NEXT:    vpsadbw (%rdx), %ymm0, %ymm0
1259; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
1260; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
1261; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1262; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
1263; AVX2-NEXT:    vmovd %xmm0, %eax
1264; AVX2-NEXT:    vzeroupper
1265; AVX2-NEXT:    retq
1266;
1267; AVX512F-LABEL: sad_nonloop_32i8:
1268; AVX512F:       # BB#0:
1269; AVX512F-NEXT:    vmovdqu (%rdi), %ymm0
1270; AVX512F-NEXT:    vpsadbw (%rdx), %ymm0, %ymm0
1271; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
1272; AVX512F-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
1273; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1274; AVX512F-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
1275; AVX512F-NEXT:    vmovd %xmm0, %eax
1276; AVX512F-NEXT:    retq
1277;
1278; AVX512BW-LABEL: sad_nonloop_32i8:
1279; AVX512BW:       # BB#0:
1280; AVX512BW-NEXT:    vmovdqu (%rdi), %ymm0
1281; AVX512BW-NEXT:    vpsadbw (%rdx), %ymm0, %ymm0
1282; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
1283; AVX512BW-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
1284; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
1285; AVX512BW-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
1286; AVX512BW-NEXT:    vmovd %xmm0, %eax
1287; AVX512BW-NEXT:    retq
1288  %v1 = load <32 x i8>, <32 x i8>* %p, align 1
1289  %z1 = zext <32 x i8> %v1 to <32 x i32>
1290  %v2 = load <32 x i8>, <32 x i8>* %q, align 1
1291  %z2 = zext <32 x i8> %v2 to <32 x i32>
1292  %sub = sub nsw <32 x i32> %z1, %z2
1293  %isneg = icmp sgt <32 x i32> %sub, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
1294  %neg = sub nsw <32 x i32> zeroinitializer, %sub
1295  %abs = select <32 x i1> %isneg, <32 x i32> %sub, <32 x i32> %neg
1296  %h32 = shufflevector <32 x i32> %abs, <32 x i32> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1297  %sum32 = add <32 x i32> %abs, %h32
1298  %h0 = shufflevector <32 x i32> %sum32, <32 x i32> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1299  %sum0 = add <32 x i32> %sum32, %h0
1300  %h1 = shufflevector <32 x i32> %sum0, <32 x i32> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1301  %sum1 = add <32 x i32> %sum0, %h1
1302  %h2 = shufflevector <32 x i32> %sum1, <32 x i32> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1303  %sum2 = add <32 x i32> %sum1, %h2
1304  %h3 = shufflevector <32 x i32> %sum2, <32 x i32> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
1305  %sum3 = add <32 x i32> %sum2, %h3
1306  %sum = extractelement <32 x i32> %sum3, i32 0
1307  ret i32 %sum
1308}
1309