1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py 2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=SSE2 3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=AVX --check-prefix=AVX1 4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2 5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512F 6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512BW 7 8define void @avg_v4i8(<4 x i8>* %a, <4 x i8>* %b) nounwind { 9; SSE2-LABEL: avg_v4i8: 10; SSE2: # %bb.0: 11; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero 12; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero 13; SSE2-NEXT: pavgb %xmm0, %xmm1 14; SSE2-NEXT: movd %xmm1, (%rax) 15; SSE2-NEXT: retq 16; 17; AVX-LABEL: avg_v4i8: 18; AVX: # %bb.0: 19; AVX-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero 20; AVX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero 21; AVX-NEXT: vpavgb %xmm0, %xmm1, %xmm0 22; AVX-NEXT: vmovd %xmm0, (%rax) 23; AVX-NEXT: retq 24 %1 = load <4 x i8>, <4 x i8>* %a 25 %2 = load <4 x i8>, <4 x i8>* %b 26 %3 = zext <4 x i8> %1 to <4 x i32> 27 %4 = zext <4 x i8> %2 to <4 x i32> 28 %5 = add nuw nsw <4 x i32> %3, <i32 1, i32 1, i32 1, i32 1> 29 %6 = add nuw nsw <4 x i32> %5, %4 30 %7 = lshr <4 x i32> %6, <i32 1, i32 1, i32 1, i32 1> 31 %8 = trunc <4 x i32> %7 to <4 x i8> 32 store <4 x i8> %8, <4 x i8>* undef, align 4 33 ret void 34} 35 36define void @avg_v8i8(<8 x i8>* %a, <8 x i8>* %b) nounwind { 37; SSE2-LABEL: avg_v8i8: 38; SSE2: # %bb.0: 39; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero 40; SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero 41; SSE2-NEXT: pavgb %xmm0, %xmm1 42; SSE2-NEXT: movq %xmm1, (%rax) 43; SSE2-NEXT: retq 44; 45; AVX-LABEL: avg_v8i8: 46; AVX: # %bb.0: 47; AVX-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 48; AVX-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero 49; AVX-NEXT: vpavgb %xmm0, %xmm1, %xmm0 50; AVX-NEXT: vmovq %xmm0, (%rax) 51; AVX-NEXT: retq 52 %1 = load <8 x i8>, <8 x i8>* %a 53 %2 = load <8 x i8>, <8 x i8>* %b 54 %3 = zext <8 x i8> %1 to <8 x i32> 55 %4 = zext <8 x i8> %2 to <8 x i32> 56 %5 = add nuw nsw <8 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 57 %6 = add nuw nsw <8 x i32> %5, %4 58 %7 = lshr <8 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 59 %8 = trunc <8 x i32> %7 to <8 x i8> 60 store <8 x i8> %8, <8 x i8>* undef, align 4 61 ret void 62} 63 64define void @avg_v16i8(<16 x i8>* %a, <16 x i8>* %b) nounwind { 65; SSE2-LABEL: avg_v16i8: 66; SSE2: # %bb.0: 67; SSE2-NEXT: movdqa (%rsi), %xmm0 68; SSE2-NEXT: pavgb (%rdi), %xmm0 69; SSE2-NEXT: movdqu %xmm0, (%rax) 70; SSE2-NEXT: retq 71; 72; AVX-LABEL: avg_v16i8: 73; AVX: # %bb.0: 74; AVX-NEXT: vmovdqa (%rsi), %xmm0 75; AVX-NEXT: vpavgb (%rdi), %xmm0, %xmm0 76; AVX-NEXT: vmovdqu %xmm0, (%rax) 77; AVX-NEXT: retq 78 %1 = load <16 x i8>, <16 x i8>* %a 79 %2 = load <16 x i8>, <16 x i8>* %b 80 %3 = zext <16 x i8> %1 to <16 x i32> 81 %4 = zext <16 x i8> %2 to <16 x i32> 82 %5 = add nuw nsw <16 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 83 %6 = add nuw nsw <16 x i32> %5, %4 84 %7 = lshr <16 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 85 %8 = trunc <16 x i32> %7 to <16 x i8> 86 store <16 x i8> %8, <16 x i8>* undef, align 4 87 ret void 88} 89 90define void @avg_v32i8(<32 x i8>* %a, <32 x i8>* %b) nounwind { 91; SSE2-LABEL: avg_v32i8: 92; SSE2: # %bb.0: 93; SSE2-NEXT: movdqa (%rsi), %xmm0 94; SSE2-NEXT: movdqa 16(%rsi), %xmm1 95; SSE2-NEXT: pavgb (%rdi), %xmm0 96; SSE2-NEXT: pavgb 16(%rdi), %xmm1 97; SSE2-NEXT: movdqu %xmm1, (%rax) 98; SSE2-NEXT: movdqu %xmm0, (%rax) 99; SSE2-NEXT: retq 100; 101; AVX1-LABEL: avg_v32i8: 102; AVX1: # %bb.0: 103; AVX1-NEXT: vmovdqa (%rsi), %xmm0 104; AVX1-NEXT: vmovdqa 16(%rsi), %xmm1 105; AVX1-NEXT: vpavgb 16(%rdi), %xmm1, %xmm1 106; AVX1-NEXT: vpavgb (%rdi), %xmm0, %xmm0 107; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 108; AVX1-NEXT: vmovups %ymm0, (%rax) 109; AVX1-NEXT: vzeroupper 110; AVX1-NEXT: retq 111; 112; AVX2-LABEL: avg_v32i8: 113; AVX2: # %bb.0: 114; AVX2-NEXT: vmovdqa (%rsi), %ymm0 115; AVX2-NEXT: vpavgb (%rdi), %ymm0, %ymm0 116; AVX2-NEXT: vmovdqu %ymm0, (%rax) 117; AVX2-NEXT: vzeroupper 118; AVX2-NEXT: retq 119; 120; AVX512-LABEL: avg_v32i8: 121; AVX512: # %bb.0: 122; AVX512-NEXT: vmovdqa (%rsi), %ymm0 123; AVX512-NEXT: vpavgb (%rdi), %ymm0, %ymm0 124; AVX512-NEXT: vmovdqu %ymm0, (%rax) 125; AVX512-NEXT: vzeroupper 126; AVX512-NEXT: retq 127 %1 = load <32 x i8>, <32 x i8>* %a 128 %2 = load <32 x i8>, <32 x i8>* %b 129 %3 = zext <32 x i8> %1 to <32 x i32> 130 %4 = zext <32 x i8> %2 to <32 x i32> 131 %5 = add nuw nsw <32 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 132 %6 = add nuw nsw <32 x i32> %5, %4 133 %7 = lshr <32 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 134 %8 = trunc <32 x i32> %7 to <32 x i8> 135 store <32 x i8> %8, <32 x i8>* undef, align 4 136 ret void 137} 138 139define void @avg_v48i8(<48 x i8>* %a, <48 x i8>* %b) nounwind { 140; SSE2-LABEL: avg_v48i8: 141; SSE2: # %bb.0: 142; SSE2-NEXT: movdqa (%rdi), %xmm1 143; SSE2-NEXT: movdqa 16(%rdi), %xmm6 144; SSE2-NEXT: movdqa 32(%rdi), %xmm11 145; SSE2-NEXT: movdqa (%rsi), %xmm12 146; SSE2-NEXT: movdqa 16(%rsi), %xmm13 147; SSE2-NEXT: movdqa 32(%rsi), %xmm0 148; SSE2-NEXT: pxor %xmm7, %xmm7 149; SSE2-NEXT: movdqa %xmm1, %xmm4 150; SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm7[8],xmm4[9],xmm7[9],xmm4[10],xmm7[10],xmm4[11],xmm7[11],xmm4[12],xmm7[12],xmm4[13],xmm7[13],xmm4[14],xmm7[14],xmm4[15],xmm7[15] 151; SSE2-NEXT: movdqa %xmm4, %xmm2 152; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm7[4],xmm2[5],xmm7[5],xmm2[6],xmm7[6],xmm2[7],xmm7[7] 153; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1],xmm4[2],xmm7[2],xmm4[3],xmm7[3] 154; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm7[0],xmm1[1],xmm7[1],xmm1[2],xmm7[2],xmm1[3],xmm7[3],xmm1[4],xmm7[4],xmm1[5],xmm7[5],xmm1[6],xmm7[6],xmm1[7],xmm7[7] 155; SSE2-NEXT: movdqa %xmm1, %xmm10 156; SSE2-NEXT: punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm7[4],xmm10[5],xmm7[5],xmm10[6],xmm7[6],xmm10[7],xmm7[7] 157; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm7[0],xmm1[1],xmm7[1],xmm1[2],xmm7[2],xmm1[3],xmm7[3] 158; SSE2-NEXT: movdqa %xmm6, %xmm5 159; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm7[8],xmm5[9],xmm7[9],xmm5[10],xmm7[10],xmm5[11],xmm7[11],xmm5[12],xmm7[12],xmm5[13],xmm7[13],xmm5[14],xmm7[14],xmm5[15],xmm7[15] 160; SSE2-NEXT: movdqa %xmm5, %xmm15 161; SSE2-NEXT: punpckhwd {{.*#+}} xmm15 = xmm15[4],xmm7[4],xmm15[5],xmm7[5],xmm15[6],xmm7[6],xmm15[7],xmm7[7] 162; SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3] 163; SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3],xmm6[4],xmm7[4],xmm6[5],xmm7[5],xmm6[6],xmm7[6],xmm6[7],xmm7[7] 164; SSE2-NEXT: movdqa %xmm6, %xmm14 165; SSE2-NEXT: punpckhwd {{.*#+}} xmm14 = xmm14[4],xmm7[4],xmm14[5],xmm7[5],xmm14[6],xmm7[6],xmm14[7],xmm7[7] 166; SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3] 167; SSE2-NEXT: movdqa %xmm12, %xmm3 168; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm7[8],xmm3[9],xmm7[9],xmm3[10],xmm7[10],xmm3[11],xmm7[11],xmm3[12],xmm7[12],xmm3[13],xmm7[13],xmm3[14],xmm7[14],xmm3[15],xmm7[15] 169; SSE2-NEXT: movdqa %xmm3, %xmm8 170; SSE2-NEXT: punpckhwd {{.*#+}} xmm8 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7] 171; SSE2-NEXT: paddd %xmm2, %xmm8 172; SSE2-NEXT: movdqa %xmm11, %xmm2 173; SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm7[8],xmm2[9],xmm7[9],xmm2[10],xmm7[10],xmm2[11],xmm7[11],xmm2[12],xmm7[12],xmm2[13],xmm7[13],xmm2[14],xmm7[14],xmm2[15],xmm7[15] 174; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm7[0],xmm3[1],xmm7[1],xmm3[2],xmm7[2],xmm3[3],xmm7[3] 175; SSE2-NEXT: paddd %xmm4, %xmm3 176; SSE2-NEXT: punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm7[0],xmm12[1],xmm7[1],xmm12[2],xmm7[2],xmm12[3],xmm7[3],xmm12[4],xmm7[4],xmm12[5],xmm7[5],xmm12[6],xmm7[6],xmm12[7],xmm7[7] 177; SSE2-NEXT: movdqa %xmm12, %xmm9 178; SSE2-NEXT: punpckhwd {{.*#+}} xmm9 = xmm9[4],xmm7[4],xmm9[5],xmm7[5],xmm9[6],xmm7[6],xmm9[7],xmm7[7] 179; SSE2-NEXT: paddd %xmm10, %xmm9 180; SSE2-NEXT: punpcklwd {{.*#+}} xmm12 = xmm12[0],xmm7[0],xmm12[1],xmm7[1],xmm12[2],xmm7[2],xmm12[3],xmm7[3] 181; SSE2-NEXT: paddd %xmm1, %xmm12 182; SSE2-NEXT: movdqa %xmm13, %xmm4 183; SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm7[8],xmm4[9],xmm7[9],xmm4[10],xmm7[10],xmm4[11],xmm7[11],xmm4[12],xmm7[12],xmm4[13],xmm7[13],xmm4[14],xmm7[14],xmm4[15],xmm7[15] 184; SSE2-NEXT: movdqa %xmm4, %xmm10 185; SSE2-NEXT: punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm7[4],xmm10[5],xmm7[5],xmm10[6],xmm7[6],xmm10[7],xmm7[7] 186; SSE2-NEXT: paddd %xmm15, %xmm10 187; SSE2-NEXT: movdqa %xmm2, %xmm15 188; SSE2-NEXT: punpckhwd {{.*#+}} xmm15 = xmm15[4],xmm7[4],xmm15[5],xmm7[5],xmm15[6],xmm7[6],xmm15[7],xmm7[7] 189; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm7[0],xmm2[1],xmm7[1],xmm2[2],xmm7[2],xmm2[3],xmm7[3] 190; SSE2-NEXT: punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm7[0],xmm11[1],xmm7[1],xmm11[2],xmm7[2],xmm11[3],xmm7[3],xmm11[4],xmm7[4],xmm11[5],xmm7[5],xmm11[6],xmm7[6],xmm11[7],xmm7[7] 191; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm7[0],xmm4[1],xmm7[1],xmm4[2],xmm7[2],xmm4[3],xmm7[3] 192; SSE2-NEXT: paddd %xmm5, %xmm4 193; SSE2-NEXT: punpcklbw {{.*#+}} xmm13 = xmm13[0],xmm7[0],xmm13[1],xmm7[1],xmm13[2],xmm7[2],xmm13[3],xmm7[3],xmm13[4],xmm7[4],xmm13[5],xmm7[5],xmm13[6],xmm7[6],xmm13[7],xmm7[7] 194; SSE2-NEXT: movdqa %xmm13, %xmm1 195; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm7[4],xmm1[5],xmm7[5],xmm1[6],xmm7[6],xmm1[7],xmm7[7] 196; SSE2-NEXT: paddd %xmm14, %xmm1 197; SSE2-NEXT: punpcklwd {{.*#+}} xmm13 = xmm13[0],xmm7[0],xmm13[1],xmm7[1],xmm13[2],xmm7[2],xmm13[3],xmm7[3] 198; SSE2-NEXT: paddd %xmm6, %xmm13 199; SSE2-NEXT: movdqa %xmm0, %xmm6 200; SSE2-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm7[8],xmm6[9],xmm7[9],xmm6[10],xmm7[10],xmm6[11],xmm7[11],xmm6[12],xmm7[12],xmm6[13],xmm7[13],xmm6[14],xmm7[14],xmm6[15],xmm7[15] 201; SSE2-NEXT: movdqa %xmm6, %xmm14 202; SSE2-NEXT: punpckhwd {{.*#+}} xmm14 = xmm14[4],xmm7[4],xmm14[5],xmm7[5],xmm14[6],xmm7[6],xmm14[7],xmm7[7] 203; SSE2-NEXT: paddd %xmm15, %xmm14 204; SSE2-NEXT: movdqa %xmm11, %xmm5 205; SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm7[4],xmm5[5],xmm7[5],xmm5[6],xmm7[6],xmm5[7],xmm7[7] 206; SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3] 207; SSE2-NEXT: paddd %xmm2, %xmm6 208; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1],xmm0[2],xmm7[2],xmm0[3],xmm7[3],xmm0[4],xmm7[4],xmm0[5],xmm7[5],xmm0[6],xmm7[6],xmm0[7],xmm7[7] 209; SSE2-NEXT: movdqa %xmm0, %xmm2 210; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm7[4],xmm2[5],xmm7[5],xmm2[6],xmm7[6],xmm2[7],xmm7[7] 211; SSE2-NEXT: paddd %xmm5, %xmm2 212; SSE2-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm7[0],xmm11[1],xmm7[1],xmm11[2],xmm7[2],xmm11[3],xmm7[3] 213; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1],xmm0[2],xmm7[2],xmm0[3],xmm7[3] 214; SSE2-NEXT: paddd %xmm11, %xmm0 215; SSE2-NEXT: pcmpeqd %xmm5, %xmm5 216; SSE2-NEXT: psubd %xmm5, %xmm8 217; SSE2-NEXT: psubd %xmm5, %xmm3 218; SSE2-NEXT: psubd %xmm5, %xmm9 219; SSE2-NEXT: psubd %xmm5, %xmm12 220; SSE2-NEXT: psubd %xmm5, %xmm10 221; SSE2-NEXT: psubd %xmm5, %xmm4 222; SSE2-NEXT: psubd %xmm5, %xmm1 223; SSE2-NEXT: psubd %xmm5, %xmm13 224; SSE2-NEXT: psubd %xmm5, %xmm14 225; SSE2-NEXT: psubd %xmm5, %xmm6 226; SSE2-NEXT: psubd %xmm5, %xmm2 227; SSE2-NEXT: psubd %xmm5, %xmm0 228; SSE2-NEXT: psrld $1, %xmm3 229; SSE2-NEXT: psrld $1, %xmm8 230; SSE2-NEXT: movdqa {{.*#+}} xmm7 = [255,255,255,255] 231; SSE2-NEXT: pand %xmm7, %xmm8 232; SSE2-NEXT: pand %xmm7, %xmm3 233; SSE2-NEXT: packuswb %xmm8, %xmm3 234; SSE2-NEXT: psrld $1, %xmm12 235; SSE2-NEXT: psrld $1, %xmm9 236; SSE2-NEXT: pand %xmm7, %xmm9 237; SSE2-NEXT: pand %xmm7, %xmm12 238; SSE2-NEXT: packuswb %xmm9, %xmm12 239; SSE2-NEXT: packuswb %xmm3, %xmm12 240; SSE2-NEXT: psrld $1, %xmm4 241; SSE2-NEXT: psrld $1, %xmm10 242; SSE2-NEXT: pand %xmm7, %xmm10 243; SSE2-NEXT: pand %xmm7, %xmm4 244; SSE2-NEXT: packuswb %xmm10, %xmm4 245; SSE2-NEXT: psrld $1, %xmm13 246; SSE2-NEXT: psrld $1, %xmm1 247; SSE2-NEXT: pand %xmm7, %xmm1 248; SSE2-NEXT: pand %xmm7, %xmm13 249; SSE2-NEXT: packuswb %xmm1, %xmm13 250; SSE2-NEXT: packuswb %xmm4, %xmm13 251; SSE2-NEXT: psrld $1, %xmm6 252; SSE2-NEXT: psrld $1, %xmm14 253; SSE2-NEXT: pand %xmm7, %xmm14 254; SSE2-NEXT: pand %xmm7, %xmm6 255; SSE2-NEXT: packuswb %xmm14, %xmm6 256; SSE2-NEXT: psrld $1, %xmm0 257; SSE2-NEXT: psrld $1, %xmm2 258; SSE2-NEXT: pand %xmm7, %xmm2 259; SSE2-NEXT: pand %xmm7, %xmm0 260; SSE2-NEXT: packuswb %xmm2, %xmm0 261; SSE2-NEXT: packuswb %xmm6, %xmm0 262; SSE2-NEXT: movdqu %xmm0, (%rax) 263; SSE2-NEXT: movdqu %xmm13, (%rax) 264; SSE2-NEXT: movdqu %xmm12, (%rax) 265; SSE2-NEXT: retq 266; 267; AVX1-LABEL: avg_v48i8: 268; AVX1: # %bb.0: 269; AVX1-NEXT: vmovdqa (%rdi), %xmm0 270; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 271; AVX1-NEXT: vmovdqa 32(%rdi), %xmm4 272; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,0,1] 273; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm5 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero 274; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[3,3,0,1] 275; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm6 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero 276; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3] 277; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm7 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero 278; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero 279; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[2,3,0,1] 280; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm15 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero 281; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[3,3,0,1] 282; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm10 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero 283; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[1,1,2,3] 284; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm14 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero 285; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm12 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero 286; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm4[2,3,0,1] 287; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero 288; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill 289; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm4[3,3,0,1] 290; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero 291; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill 292; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm4[1,1,2,3] 293; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero 294; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill 295; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero 296; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill 297; AVX1-NEXT: vmovdqa (%rsi), %xmm0 298; AVX1-NEXT: vmovdqa 16(%rsi), %xmm4 299; AVX1-NEXT: vmovdqa 32(%rsi), %xmm3 300; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm4[2,3,0,1] 301; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero 302; AVX1-NEXT: vpaddd %xmm2, %xmm5, %xmm13 303; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[3,3,0,1] 304; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero 305; AVX1-NEXT: vpaddd %xmm5, %xmm6, %xmm11 306; AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[1,1,2,3] 307; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero 308; AVX1-NEXT: vpaddd %xmm6, %xmm7, %xmm9 309; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero 310; AVX1-NEXT: vpaddd %xmm4, %xmm1, %xmm8 311; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,0,1] 312; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero 313; AVX1-NEXT: vpaddd %xmm4, %xmm15, %xmm15 314; AVX1-NEXT: vpshufd {{.*#+}} xmm7 = xmm0[3,3,0,1] 315; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero 316; AVX1-NEXT: vpaddd %xmm7, %xmm10, %xmm7 317; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3] 318; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero 319; AVX1-NEXT: vpaddd %xmm2, %xmm14, %xmm14 320; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero 321; AVX1-NEXT: vpaddd %xmm0, %xmm12, %xmm12 322; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[2,3,0,1] 323; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero 324; AVX1-NEXT: vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm5 # 16-byte Folded Reload 325; AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[3,3,0,1] 326; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero 327; AVX1-NEXT: vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %xmm6, %xmm6 # 16-byte Folded Reload 328; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm3[1,1,2,3] 329; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero 330; AVX1-NEXT: vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload 331; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero 332; AVX1-NEXT: vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload 333; AVX1-NEXT: vpcmpeqd %xmm4, %xmm4, %xmm4 334; AVX1-NEXT: vpsubd %xmm4, %xmm13, %xmm10 335; AVX1-NEXT: vpsubd %xmm4, %xmm11, %xmm11 336; AVX1-NEXT: vpsubd %xmm4, %xmm9, %xmm9 337; AVX1-NEXT: vpsubd %xmm4, %xmm8, %xmm8 338; AVX1-NEXT: vpsubd %xmm4, %xmm15, %xmm13 339; AVX1-NEXT: vpsubd %xmm4, %xmm7, %xmm7 340; AVX1-NEXT: vpsubd %xmm4, %xmm14, %xmm0 341; AVX1-NEXT: vpsubd %xmm4, %xmm12, %xmm2 342; AVX1-NEXT: vpsubd %xmm4, %xmm5, %xmm5 343; AVX1-NEXT: vpsubd %xmm4, %xmm6, %xmm6 344; AVX1-NEXT: vpsubd %xmm4, %xmm1, %xmm1 345; AVX1-NEXT: vpsubd %xmm4, %xmm3, %xmm3 346; AVX1-NEXT: vpsrld $1, %xmm3, %xmm3 347; AVX1-NEXT: vpsrld $1, %xmm1, %xmm1 348; AVX1-NEXT: vpackusdw %xmm1, %xmm3, %xmm1 349; AVX1-NEXT: vpsrld $1, %xmm6, %xmm3 350; AVX1-NEXT: vpsrld $1, %xmm5, %xmm4 351; AVX1-NEXT: vpackusdw %xmm3, %xmm4, %xmm3 352; AVX1-NEXT: vpsrld $1, %xmm2, %xmm2 353; AVX1-NEXT: vpsrld $1, %xmm0, %xmm0 354; AVX1-NEXT: vpackusdw %xmm0, %xmm2, %xmm0 355; AVX1-NEXT: vpsrld $1, %xmm7, %xmm2 356; AVX1-NEXT: vpsrld $1, %xmm13, %xmm4 357; AVX1-NEXT: vpackusdw %xmm2, %xmm4, %xmm2 358; AVX1-NEXT: vpsrld $1, %xmm8, %xmm4 359; AVX1-NEXT: vpsrld $1, %xmm9, %xmm5 360; AVX1-NEXT: vpackusdw %xmm5, %xmm4, %xmm4 361; AVX1-NEXT: vpsrld $1, %xmm11, %xmm5 362; AVX1-NEXT: vpsrld $1, %xmm10, %xmm6 363; AVX1-NEXT: vpackusdw %xmm5, %xmm6, %xmm5 364; AVX1-NEXT: vmovdqa {{.*#+}} xmm6 = [255,255,255,255,255,255,255,255] 365; AVX1-NEXT: vpand %xmm6, %xmm5, %xmm5 366; AVX1-NEXT: vpand %xmm6, %xmm4, %xmm4 367; AVX1-NEXT: vpackuswb %xmm5, %xmm4, %xmm4 368; AVX1-NEXT: vpand %xmm6, %xmm2, %xmm2 369; AVX1-NEXT: vpand %xmm6, %xmm0, %xmm0 370; AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0 371; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 372; AVX1-NEXT: vpand %xmm6, %xmm3, %xmm2 373; AVX1-NEXT: vpand %xmm6, %xmm1, %xmm1 374; AVX1-NEXT: vpackuswb %xmm2, %xmm1, %xmm1 375; AVX1-NEXT: vmovdqu %xmm1, (%rax) 376; AVX1-NEXT: vmovups %ymm0, (%rax) 377; AVX1-NEXT: vzeroupper 378; AVX1-NEXT: retq 379; 380; AVX2-LABEL: avg_v48i8: 381; AVX2: # %bb.0: 382; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 383; AVX2-NEXT: vpbroadcastq 24(%rdi), %ymm1 384; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero 385; AVX2-NEXT: vmovdqa (%rdi), %xmm2 386; AVX2-NEXT: vmovdqa 32(%rdi), %xmm3 387; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm4 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero 388; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1] 389; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero 390; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm5 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero 391; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1] 392; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero 393; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm6 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 394; AVX2-NEXT: vpaddd %ymm6, %ymm0, %ymm0 395; AVX2-NEXT: vpbroadcastq 24(%rsi), %ymm6 396; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero,xmm6[4],zero,zero,zero,xmm6[5],zero,zero,zero,xmm6[6],zero,zero,zero,xmm6[7],zero,zero,zero 397; AVX2-NEXT: vpaddd %ymm6, %ymm1, %ymm1 398; AVX2-NEXT: vmovdqa (%rsi), %xmm6 399; AVX2-NEXT: vmovdqa 32(%rsi), %xmm7 400; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero,xmm6[4],zero,zero,zero,xmm6[5],zero,zero,zero,xmm6[6],zero,zero,zero,xmm6[7],zero,zero,zero 401; AVX2-NEXT: vpaddd %ymm8, %ymm4, %ymm4 402; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[2,3,0,1] 403; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero,xmm6[4],zero,zero,zero,xmm6[5],zero,zero,zero,xmm6[6],zero,zero,zero,xmm6[7],zero,zero,zero 404; AVX2-NEXT: vpaddd %ymm6, %ymm2, %ymm2 405; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm6 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero,xmm7[4],zero,zero,zero,xmm7[5],zero,zero,zero,xmm7[6],zero,zero,zero,xmm7[7],zero,zero,zero 406; AVX2-NEXT: vpaddd %ymm6, %ymm5, %ymm5 407; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm7[2,3,0,1] 408; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero,xmm6[4],zero,zero,zero,xmm6[5],zero,zero,zero,xmm6[6],zero,zero,zero,xmm6[7],zero,zero,zero 409; AVX2-NEXT: vpaddd %ymm6, %ymm3, %ymm3 410; AVX2-NEXT: vpcmpeqd %ymm6, %ymm6, %ymm6 411; AVX2-NEXT: vpsubd %ymm6, %ymm0, %ymm7 412; AVX2-NEXT: vpsubd %ymm6, %ymm1, %ymm1 413; AVX2-NEXT: vpsubd %ymm6, %ymm4, %ymm4 414; AVX2-NEXT: vpsubd %ymm6, %ymm2, %ymm2 415; AVX2-NEXT: vpsubd %ymm6, %ymm5, %ymm5 416; AVX2-NEXT: vpsubd %ymm6, %ymm3, %ymm0 417; AVX2-NEXT: vpsrld $1, %ymm0, %ymm0 418; AVX2-NEXT: vpsrld $1, %ymm5, %ymm3 419; AVX2-NEXT: vpsrld $1, %ymm2, %ymm2 420; AVX2-NEXT: vpsrld $1, %ymm4, %ymm4 421; AVX2-NEXT: vpsrld $1, %ymm1, %ymm1 422; AVX2-NEXT: vpsrld $1, %ymm7, %ymm5 423; AVX2-NEXT: vextracti128 $1, %ymm5, %xmm6 424; AVX2-NEXT: vpackusdw %xmm6, %xmm5, %xmm5 425; AVX2-NEXT: vmovdqa {{.*#+}} xmm6 = [255,255,255,255,255,255,255,255] 426; AVX2-NEXT: vpand %xmm6, %xmm5, %xmm5 427; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm7 428; AVX2-NEXT: vpackusdw %xmm7, %xmm1, %xmm1 429; AVX2-NEXT: vpand %xmm6, %xmm1, %xmm1 430; AVX2-NEXT: vpackuswb %xmm1, %xmm5, %xmm1 431; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm5 432; AVX2-NEXT: vpackusdw %xmm5, %xmm4, %xmm4 433; AVX2-NEXT: vpand %xmm6, %xmm4, %xmm4 434; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm5 435; AVX2-NEXT: vpackusdw %xmm5, %xmm2, %xmm2 436; AVX2-NEXT: vpand %xmm6, %xmm2, %xmm2 437; AVX2-NEXT: vpackuswb %xmm2, %xmm4, %xmm2 438; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1 439; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm2 440; AVX2-NEXT: vpackusdw %xmm2, %xmm3, %xmm2 441; AVX2-NEXT: vpand %xmm6, %xmm2, %xmm2 442; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm3 443; AVX2-NEXT: vpackusdw %xmm3, %xmm0, %xmm0 444; AVX2-NEXT: vpand %xmm6, %xmm0, %xmm0 445; AVX2-NEXT: vpackuswb %xmm0, %xmm2, %xmm0 446; AVX2-NEXT: vmovdqu %xmm0, (%rax) 447; AVX2-NEXT: vmovdqu %ymm1, (%rax) 448; AVX2-NEXT: vzeroupper 449; AVX2-NEXT: retq 450; 451; AVX512F-LABEL: avg_v48i8: 452; AVX512F: # %bb.0: 453; AVX512F-NEXT: vmovdqa (%rdi), %xmm0 454; AVX512F-NEXT: vmovdqa 16(%rdi), %xmm1 455; AVX512F-NEXT: vmovdqa 32(%rdi), %xmm2 456; AVX512F-NEXT: vpavgb 16(%rsi), %xmm1, %xmm1 457; AVX512F-NEXT: vpavgb (%rsi), %xmm0, %xmm0 458; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 459; AVX512F-NEXT: vpavgb 32(%rsi), %xmm2, %xmm1 460; AVX512F-NEXT: vmovdqu %xmm1, (%rax) 461; AVX512F-NEXT: vmovdqu %ymm0, (%rax) 462; AVX512F-NEXT: vzeroupper 463; AVX512F-NEXT: retq 464; 465; AVX512BW-LABEL: avg_v48i8: 466; AVX512BW: # %bb.0: 467; AVX512BW-NEXT: vmovdqa (%rdi), %xmm0 468; AVX512BW-NEXT: vmovdqa 16(%rdi), %xmm1 469; AVX512BW-NEXT: vmovdqa 32(%rdi), %xmm2 470; AVX512BW-NEXT: vpavgb 16(%rsi), %xmm1, %xmm1 471; AVX512BW-NEXT: vpavgb (%rsi), %xmm0, %xmm0 472; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 473; AVX512BW-NEXT: vpavgb 32(%rsi), %xmm2, %xmm1 474; AVX512BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm1 475; AVX512BW-NEXT: vmovdqu %ymm0, (%rax) 476; AVX512BW-NEXT: vextracti32x4 $2, %zmm1, (%rax) 477; AVX512BW-NEXT: vzeroupper 478; AVX512BW-NEXT: retq 479 %1 = load <48 x i8>, <48 x i8>* %a 480 %2 = load <48 x i8>, <48 x i8>* %b 481 %3 = zext <48 x i8> %1 to <48 x i32> 482 %4 = zext <48 x i8> %2 to <48 x i32> 483 %5 = add nuw nsw <48 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 484 %6 = add nuw nsw <48 x i32> %5, %4 485 %7 = lshr <48 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 486 %8 = trunc <48 x i32> %7 to <48 x i8> 487 store <48 x i8> %8, <48 x i8>* undef, align 4 488 ret void 489} 490 491define void @avg_v64i8(<64 x i8>* %a, <64 x i8>* %b) nounwind { 492; SSE2-LABEL: avg_v64i8: 493; SSE2: # %bb.0: 494; SSE2-NEXT: movdqa (%rsi), %xmm0 495; SSE2-NEXT: movdqa 16(%rsi), %xmm1 496; SSE2-NEXT: movdqa 32(%rsi), %xmm2 497; SSE2-NEXT: movdqa 48(%rsi), %xmm3 498; SSE2-NEXT: pavgb (%rdi), %xmm0 499; SSE2-NEXT: pavgb 16(%rdi), %xmm1 500; SSE2-NEXT: pavgb 32(%rdi), %xmm2 501; SSE2-NEXT: pavgb 48(%rdi), %xmm3 502; SSE2-NEXT: movdqu %xmm3, (%rax) 503; SSE2-NEXT: movdqu %xmm2, (%rax) 504; SSE2-NEXT: movdqu %xmm1, (%rax) 505; SSE2-NEXT: movdqu %xmm0, (%rax) 506; SSE2-NEXT: retq 507; 508; AVX1-LABEL: avg_v64i8: 509; AVX1: # %bb.0: 510; AVX1-NEXT: vmovdqa (%rsi), %xmm0 511; AVX1-NEXT: vmovdqa 16(%rsi), %xmm1 512; AVX1-NEXT: vmovdqa 32(%rsi), %xmm2 513; AVX1-NEXT: vmovdqa 48(%rsi), %xmm3 514; AVX1-NEXT: vpavgb 16(%rdi), %xmm1, %xmm1 515; AVX1-NEXT: vpavgb (%rdi), %xmm0, %xmm0 516; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 517; AVX1-NEXT: vpavgb 48(%rdi), %xmm3, %xmm1 518; AVX1-NEXT: vpavgb 32(%rdi), %xmm2, %xmm2 519; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1 520; AVX1-NEXT: vmovups %ymm1, (%rax) 521; AVX1-NEXT: vmovups %ymm0, (%rax) 522; AVX1-NEXT: vzeroupper 523; AVX1-NEXT: retq 524; 525; AVX2-LABEL: avg_v64i8: 526; AVX2: # %bb.0: 527; AVX2-NEXT: vmovdqa (%rsi), %ymm0 528; AVX2-NEXT: vmovdqa 32(%rsi), %ymm1 529; AVX2-NEXT: vpavgb (%rdi), %ymm0, %ymm0 530; AVX2-NEXT: vpavgb 32(%rdi), %ymm1, %ymm1 531; AVX2-NEXT: vmovdqu %ymm1, (%rax) 532; AVX2-NEXT: vmovdqu %ymm0, (%rax) 533; AVX2-NEXT: vzeroupper 534; AVX2-NEXT: retq 535; 536; AVX512F-LABEL: avg_v64i8: 537; AVX512F: # %bb.0: 538; AVX512F-NEXT: vmovdqa (%rsi), %ymm0 539; AVX512F-NEXT: vmovdqa 32(%rsi), %ymm1 540; AVX512F-NEXT: vpavgb (%rdi), %ymm0, %ymm0 541; AVX512F-NEXT: vpavgb 32(%rdi), %ymm1, %ymm1 542; AVX512F-NEXT: vmovdqu %ymm1, (%rax) 543; AVX512F-NEXT: vmovdqu %ymm0, (%rax) 544; AVX512F-NEXT: vzeroupper 545; AVX512F-NEXT: retq 546; 547; AVX512BW-LABEL: avg_v64i8: 548; AVX512BW: # %bb.0: 549; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm0 550; AVX512BW-NEXT: vpavgb (%rdi), %zmm0, %zmm0 551; AVX512BW-NEXT: vmovdqu64 %zmm0, (%rax) 552; AVX512BW-NEXT: vzeroupper 553; AVX512BW-NEXT: retq 554 %1 = load <64 x i8>, <64 x i8>* %a 555 %2 = load <64 x i8>, <64 x i8>* %b 556 %3 = zext <64 x i8> %1 to <64 x i32> 557 %4 = zext <64 x i8> %2 to <64 x i32> 558 %5 = add nuw nsw <64 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 559 %6 = add nuw nsw <64 x i32> %5, %4 560 %7 = lshr <64 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 561 %8 = trunc <64 x i32> %7 to <64 x i8> 562 store <64 x i8> %8, <64 x i8>* undef, align 4 563 ret void 564} 565 566define void @avg_v4i16(<4 x i16>* %a, <4 x i16>* %b) nounwind { 567; SSE2-LABEL: avg_v4i16: 568; SSE2: # %bb.0: 569; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero 570; SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero 571; SSE2-NEXT: pavgw %xmm0, %xmm1 572; SSE2-NEXT: movq %xmm1, (%rax) 573; SSE2-NEXT: retq 574; 575; AVX-LABEL: avg_v4i16: 576; AVX: # %bb.0: 577; AVX-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 578; AVX-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero 579; AVX-NEXT: vpavgw %xmm0, %xmm1, %xmm0 580; AVX-NEXT: vmovq %xmm0, (%rax) 581; AVX-NEXT: retq 582 %1 = load <4 x i16>, <4 x i16>* %a 583 %2 = load <4 x i16>, <4 x i16>* %b 584 %3 = zext <4 x i16> %1 to <4 x i32> 585 %4 = zext <4 x i16> %2 to <4 x i32> 586 %5 = add nuw nsw <4 x i32> %3, <i32 1, i32 1, i32 1, i32 1> 587 %6 = add nuw nsw <4 x i32> %5, %4 588 %7 = lshr <4 x i32> %6, <i32 1, i32 1, i32 1, i32 1> 589 %8 = trunc <4 x i32> %7 to <4 x i16> 590 store <4 x i16> %8, <4 x i16>* undef, align 4 591 ret void 592} 593 594define void @avg_v8i16(<8 x i16>* %a, <8 x i16>* %b) nounwind { 595; SSE2-LABEL: avg_v8i16: 596; SSE2: # %bb.0: 597; SSE2-NEXT: movdqa (%rsi), %xmm0 598; SSE2-NEXT: pavgw (%rdi), %xmm0 599; SSE2-NEXT: movdqu %xmm0, (%rax) 600; SSE2-NEXT: retq 601; 602; AVX-LABEL: avg_v8i16: 603; AVX: # %bb.0: 604; AVX-NEXT: vmovdqa (%rsi), %xmm0 605; AVX-NEXT: vpavgw (%rdi), %xmm0, %xmm0 606; AVX-NEXT: vmovdqu %xmm0, (%rax) 607; AVX-NEXT: retq 608 %1 = load <8 x i16>, <8 x i16>* %a 609 %2 = load <8 x i16>, <8 x i16>* %b 610 %3 = zext <8 x i16> %1 to <8 x i32> 611 %4 = zext <8 x i16> %2 to <8 x i32> 612 %5 = add nuw nsw <8 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 613 %6 = add nuw nsw <8 x i32> %5, %4 614 %7 = lshr <8 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 615 %8 = trunc <8 x i32> %7 to <8 x i16> 616 store <8 x i16> %8, <8 x i16>* undef, align 4 617 ret void 618} 619 620define void @avg_v16i16(<16 x i16>* %a, <16 x i16>* %b) nounwind { 621; SSE2-LABEL: avg_v16i16: 622; SSE2: # %bb.0: 623; SSE2-NEXT: movdqa (%rsi), %xmm0 624; SSE2-NEXT: movdqa 16(%rsi), %xmm1 625; SSE2-NEXT: pavgw (%rdi), %xmm0 626; SSE2-NEXT: pavgw 16(%rdi), %xmm1 627; SSE2-NEXT: movdqu %xmm1, (%rax) 628; SSE2-NEXT: movdqu %xmm0, (%rax) 629; SSE2-NEXT: retq 630; 631; AVX1-LABEL: avg_v16i16: 632; AVX1: # %bb.0: 633; AVX1-NEXT: vmovdqa (%rsi), %xmm0 634; AVX1-NEXT: vmovdqa 16(%rsi), %xmm1 635; AVX1-NEXT: vpavgw 16(%rdi), %xmm1, %xmm1 636; AVX1-NEXT: vpavgw (%rdi), %xmm0, %xmm0 637; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 638; AVX1-NEXT: vmovups %ymm0, (%rax) 639; AVX1-NEXT: vzeroupper 640; AVX1-NEXT: retq 641; 642; AVX2-LABEL: avg_v16i16: 643; AVX2: # %bb.0: 644; AVX2-NEXT: vmovdqa (%rsi), %ymm0 645; AVX2-NEXT: vpavgw (%rdi), %ymm0, %ymm0 646; AVX2-NEXT: vmovdqu %ymm0, (%rax) 647; AVX2-NEXT: vzeroupper 648; AVX2-NEXT: retq 649; 650; AVX512-LABEL: avg_v16i16: 651; AVX512: # %bb.0: 652; AVX512-NEXT: vmovdqa (%rsi), %ymm0 653; AVX512-NEXT: vpavgw (%rdi), %ymm0, %ymm0 654; AVX512-NEXT: vmovdqu %ymm0, (%rax) 655; AVX512-NEXT: vzeroupper 656; AVX512-NEXT: retq 657 %1 = load <16 x i16>, <16 x i16>* %a 658 %2 = load <16 x i16>, <16 x i16>* %b 659 %3 = zext <16 x i16> %1 to <16 x i32> 660 %4 = zext <16 x i16> %2 to <16 x i32> 661 %5 = add nuw nsw <16 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 662 %6 = add nuw nsw <16 x i32> %5, %4 663 %7 = lshr <16 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 664 %8 = trunc <16 x i32> %7 to <16 x i16> 665 store <16 x i16> %8, <16 x i16>* undef, align 4 666 ret void 667} 668 669define void @avg_v32i16(<32 x i16>* %a, <32 x i16>* %b) nounwind { 670; SSE2-LABEL: avg_v32i16: 671; SSE2: # %bb.0: 672; SSE2-NEXT: movdqa (%rsi), %xmm0 673; SSE2-NEXT: movdqa 16(%rsi), %xmm1 674; SSE2-NEXT: movdqa 32(%rsi), %xmm2 675; SSE2-NEXT: movdqa 48(%rsi), %xmm3 676; SSE2-NEXT: pavgw (%rdi), %xmm0 677; SSE2-NEXT: pavgw 16(%rdi), %xmm1 678; SSE2-NEXT: pavgw 32(%rdi), %xmm2 679; SSE2-NEXT: pavgw 48(%rdi), %xmm3 680; SSE2-NEXT: movdqu %xmm3, (%rax) 681; SSE2-NEXT: movdqu %xmm2, (%rax) 682; SSE2-NEXT: movdqu %xmm1, (%rax) 683; SSE2-NEXT: movdqu %xmm0, (%rax) 684; SSE2-NEXT: retq 685; 686; AVX1-LABEL: avg_v32i16: 687; AVX1: # %bb.0: 688; AVX1-NEXT: vmovdqa (%rsi), %xmm0 689; AVX1-NEXT: vmovdqa 16(%rsi), %xmm1 690; AVX1-NEXT: vmovdqa 32(%rsi), %xmm2 691; AVX1-NEXT: vmovdqa 48(%rsi), %xmm3 692; AVX1-NEXT: vpavgw 16(%rdi), %xmm1, %xmm1 693; AVX1-NEXT: vpavgw (%rdi), %xmm0, %xmm0 694; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 695; AVX1-NEXT: vpavgw 48(%rdi), %xmm3, %xmm1 696; AVX1-NEXT: vpavgw 32(%rdi), %xmm2, %xmm2 697; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1 698; AVX1-NEXT: vmovups %ymm1, (%rax) 699; AVX1-NEXT: vmovups %ymm0, (%rax) 700; AVX1-NEXT: vzeroupper 701; AVX1-NEXT: retq 702; 703; AVX2-LABEL: avg_v32i16: 704; AVX2: # %bb.0: 705; AVX2-NEXT: vmovdqa (%rsi), %ymm0 706; AVX2-NEXT: vmovdqa 32(%rsi), %ymm1 707; AVX2-NEXT: vpavgw (%rdi), %ymm0, %ymm0 708; AVX2-NEXT: vpavgw 32(%rdi), %ymm1, %ymm1 709; AVX2-NEXT: vmovdqu %ymm1, (%rax) 710; AVX2-NEXT: vmovdqu %ymm0, (%rax) 711; AVX2-NEXT: vzeroupper 712; AVX2-NEXT: retq 713; 714; AVX512F-LABEL: avg_v32i16: 715; AVX512F: # %bb.0: 716; AVX512F-NEXT: vmovdqa (%rsi), %ymm0 717; AVX512F-NEXT: vmovdqa 32(%rsi), %ymm1 718; AVX512F-NEXT: vpavgw (%rdi), %ymm0, %ymm0 719; AVX512F-NEXT: vpavgw 32(%rdi), %ymm1, %ymm1 720; AVX512F-NEXT: vmovdqu %ymm1, (%rax) 721; AVX512F-NEXT: vmovdqu %ymm0, (%rax) 722; AVX512F-NEXT: vzeroupper 723; AVX512F-NEXT: retq 724; 725; AVX512BW-LABEL: avg_v32i16: 726; AVX512BW: # %bb.0: 727; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm0 728; AVX512BW-NEXT: vpavgw (%rdi), %zmm0, %zmm0 729; AVX512BW-NEXT: vmovdqu64 %zmm0, (%rax) 730; AVX512BW-NEXT: vzeroupper 731; AVX512BW-NEXT: retq 732 %1 = load <32 x i16>, <32 x i16>* %a 733 %2 = load <32 x i16>, <32 x i16>* %b 734 %3 = zext <32 x i16> %1 to <32 x i32> 735 %4 = zext <32 x i16> %2 to <32 x i32> 736 %5 = add nuw nsw <32 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 737 %6 = add nuw nsw <32 x i32> %5, %4 738 %7 = lshr <32 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 739 %8 = trunc <32 x i32> %7 to <32 x i16> 740 store <32 x i16> %8, <32 x i16>* undef, align 4 741 ret void 742} 743 744define void @avg_v4i8_2(<4 x i8>* %a, <4 x i8>* %b) nounwind { 745; SSE2-LABEL: avg_v4i8_2: 746; SSE2: # %bb.0: 747; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero 748; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero 749; SSE2-NEXT: pavgb %xmm0, %xmm1 750; SSE2-NEXT: movd %xmm1, (%rax) 751; SSE2-NEXT: retq 752; 753; AVX-LABEL: avg_v4i8_2: 754; AVX: # %bb.0: 755; AVX-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero 756; AVX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero 757; AVX-NEXT: vpavgb %xmm1, %xmm0, %xmm0 758; AVX-NEXT: vmovd %xmm0, (%rax) 759; AVX-NEXT: retq 760 %1 = load <4 x i8>, <4 x i8>* %a 761 %2 = load <4 x i8>, <4 x i8>* %b 762 %3 = zext <4 x i8> %1 to <4 x i32> 763 %4 = zext <4 x i8> %2 to <4 x i32> 764 %5 = add nuw nsw <4 x i32> %3, %4 765 %6 = add nuw nsw <4 x i32> %5, <i32 1, i32 1, i32 1, i32 1> 766 %7 = lshr <4 x i32> %6, <i32 1, i32 1, i32 1, i32 1> 767 %8 = trunc <4 x i32> %7 to <4 x i8> 768 store <4 x i8> %8, <4 x i8>* undef, align 4 769 ret void 770} 771 772define void @avg_v8i8_2(<8 x i8>* %a, <8 x i8>* %b) nounwind { 773; SSE2-LABEL: avg_v8i8_2: 774; SSE2: # %bb.0: 775; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero 776; SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero 777; SSE2-NEXT: pavgb %xmm0, %xmm1 778; SSE2-NEXT: movq %xmm1, (%rax) 779; SSE2-NEXT: retq 780; 781; AVX-LABEL: avg_v8i8_2: 782; AVX: # %bb.0: 783; AVX-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 784; AVX-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero 785; AVX-NEXT: vpavgb %xmm1, %xmm0, %xmm0 786; AVX-NEXT: vmovq %xmm0, (%rax) 787; AVX-NEXT: retq 788 %1 = load <8 x i8>, <8 x i8>* %a 789 %2 = load <8 x i8>, <8 x i8>* %b 790 %3 = zext <8 x i8> %1 to <8 x i32> 791 %4 = zext <8 x i8> %2 to <8 x i32> 792 %5 = add nuw nsw <8 x i32> %3, %4 793 %6 = add nuw nsw <8 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 794 %7 = lshr <8 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 795 %8 = trunc <8 x i32> %7 to <8 x i8> 796 store <8 x i8> %8, <8 x i8>* undef, align 4 797 ret void 798} 799 800define void @avg_v16i8_2(<16 x i8>* %a, <16 x i8>* %b) nounwind { 801; SSE2-LABEL: avg_v16i8_2: 802; SSE2: # %bb.0: 803; SSE2-NEXT: movdqa (%rdi), %xmm0 804; SSE2-NEXT: pavgb (%rsi), %xmm0 805; SSE2-NEXT: movdqu %xmm0, (%rax) 806; SSE2-NEXT: retq 807; 808; AVX-LABEL: avg_v16i8_2: 809; AVX: # %bb.0: 810; AVX-NEXT: vmovdqa (%rdi), %xmm0 811; AVX-NEXT: vpavgb (%rsi), %xmm0, %xmm0 812; AVX-NEXT: vmovdqu %xmm0, (%rax) 813; AVX-NEXT: retq 814 %1 = load <16 x i8>, <16 x i8>* %a 815 %2 = load <16 x i8>, <16 x i8>* %b 816 %3 = zext <16 x i8> %1 to <16 x i32> 817 %4 = zext <16 x i8> %2 to <16 x i32> 818 %5 = add nuw nsw <16 x i32> %3, %4 819 %6 = add nuw nsw <16 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 820 %7 = lshr <16 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 821 %8 = trunc <16 x i32> %7 to <16 x i8> 822 store <16 x i8> %8, <16 x i8>* undef, align 4 823 ret void 824} 825 826define void @avg_v32i8_2(<32 x i8>* %a, <32 x i8>* %b) nounwind { 827; SSE2-LABEL: avg_v32i8_2: 828; SSE2: # %bb.0: 829; SSE2-NEXT: movdqa (%rdi), %xmm0 830; SSE2-NEXT: movdqa 16(%rdi), %xmm1 831; SSE2-NEXT: pavgb (%rsi), %xmm0 832; SSE2-NEXT: pavgb 16(%rsi), %xmm1 833; SSE2-NEXT: movdqu %xmm1, (%rax) 834; SSE2-NEXT: movdqu %xmm0, (%rax) 835; SSE2-NEXT: retq 836; 837; AVX1-LABEL: avg_v32i8_2: 838; AVX1: # %bb.0: 839; AVX1-NEXT: vmovdqa (%rdi), %xmm0 840; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 841; AVX1-NEXT: vpavgb 16(%rsi), %xmm1, %xmm1 842; AVX1-NEXT: vpavgb (%rsi), %xmm0, %xmm0 843; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 844; AVX1-NEXT: vmovups %ymm0, (%rax) 845; AVX1-NEXT: vzeroupper 846; AVX1-NEXT: retq 847; 848; AVX2-LABEL: avg_v32i8_2: 849; AVX2: # %bb.0: 850; AVX2-NEXT: vmovdqa (%rdi), %ymm0 851; AVX2-NEXT: vpavgb (%rsi), %ymm0, %ymm0 852; AVX2-NEXT: vmovdqu %ymm0, (%rax) 853; AVX2-NEXT: vzeroupper 854; AVX2-NEXT: retq 855; 856; AVX512-LABEL: avg_v32i8_2: 857; AVX512: # %bb.0: 858; AVX512-NEXT: vmovdqa (%rdi), %ymm0 859; AVX512-NEXT: vpavgb (%rsi), %ymm0, %ymm0 860; AVX512-NEXT: vmovdqu %ymm0, (%rax) 861; AVX512-NEXT: vzeroupper 862; AVX512-NEXT: retq 863 %1 = load <32 x i8>, <32 x i8>* %a 864 %2 = load <32 x i8>, <32 x i8>* %b 865 %3 = zext <32 x i8> %1 to <32 x i32> 866 %4 = zext <32 x i8> %2 to <32 x i32> 867 %5 = add nuw nsw <32 x i32> %3, %4 868 %6 = add nuw nsw <32 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 869 %7 = lshr <32 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 870 %8 = trunc <32 x i32> %7 to <32 x i8> 871 store <32 x i8> %8, <32 x i8>* undef, align 4 872 ret void 873} 874 875define void @avg_v64i8_2(<64 x i8>* %a, <64 x i8>* %b) nounwind { 876; SSE2-LABEL: avg_v64i8_2: 877; SSE2: # %bb.0: 878; SSE2-NEXT: movdqa (%rsi), %xmm0 879; SSE2-NEXT: movdqa 16(%rsi), %xmm1 880; SSE2-NEXT: movdqa 32(%rsi), %xmm2 881; SSE2-NEXT: movdqa 48(%rsi), %xmm3 882; SSE2-NEXT: pavgb %xmm0, %xmm0 883; SSE2-NEXT: pavgb %xmm1, %xmm1 884; SSE2-NEXT: pavgb %xmm2, %xmm2 885; SSE2-NEXT: pavgb %xmm3, %xmm3 886; SSE2-NEXT: movdqu %xmm3, (%rax) 887; SSE2-NEXT: movdqu %xmm2, (%rax) 888; SSE2-NEXT: movdqu %xmm1, (%rax) 889; SSE2-NEXT: movdqu %xmm0, (%rax) 890; SSE2-NEXT: retq 891; 892; AVX1-LABEL: avg_v64i8_2: 893; AVX1: # %bb.0: 894; AVX1-NEXT: vmovdqa (%rsi), %xmm0 895; AVX1-NEXT: vmovdqa 16(%rsi), %xmm1 896; AVX1-NEXT: vmovdqa 32(%rsi), %xmm2 897; AVX1-NEXT: vmovdqa 48(%rsi), %xmm3 898; AVX1-NEXT: vpavgb %xmm0, %xmm0, %xmm0 899; AVX1-NEXT: vpavgb %xmm1, %xmm1, %xmm1 900; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 901; AVX1-NEXT: vpavgb %xmm2, %xmm2, %xmm1 902; AVX1-NEXT: vpavgb %xmm3, %xmm3, %xmm2 903; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 904; AVX1-NEXT: vmovups %ymm1, (%rax) 905; AVX1-NEXT: vmovups %ymm0, (%rax) 906; AVX1-NEXT: vzeroupper 907; AVX1-NEXT: retq 908; 909; AVX2-LABEL: avg_v64i8_2: 910; AVX2: # %bb.0: 911; AVX2-NEXT: vmovdqa (%rsi), %ymm0 912; AVX2-NEXT: vmovdqa 32(%rsi), %ymm1 913; AVX2-NEXT: vpavgb %ymm0, %ymm0, %ymm0 914; AVX2-NEXT: vpavgb %ymm1, %ymm1, %ymm1 915; AVX2-NEXT: vmovdqu %ymm1, (%rax) 916; AVX2-NEXT: vmovdqu %ymm0, (%rax) 917; AVX2-NEXT: vzeroupper 918; AVX2-NEXT: retq 919; 920; AVX512F-LABEL: avg_v64i8_2: 921; AVX512F: # %bb.0: 922; AVX512F-NEXT: vmovdqa (%rsi), %ymm0 923; AVX512F-NEXT: vmovdqa 32(%rsi), %ymm1 924; AVX512F-NEXT: vpavgb %ymm0, %ymm0, %ymm0 925; AVX512F-NEXT: vpavgb %ymm1, %ymm1, %ymm1 926; AVX512F-NEXT: vmovdqu %ymm1, (%rax) 927; AVX512F-NEXT: vmovdqu %ymm0, (%rax) 928; AVX512F-NEXT: vzeroupper 929; AVX512F-NEXT: retq 930; 931; AVX512BW-LABEL: avg_v64i8_2: 932; AVX512BW: # %bb.0: 933; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm0 934; AVX512BW-NEXT: vpavgb %zmm0, %zmm0, %zmm0 935; AVX512BW-NEXT: vmovdqu64 %zmm0, (%rax) 936; AVX512BW-NEXT: vzeroupper 937; AVX512BW-NEXT: retq 938 %1 = load <64 x i8>, <64 x i8>* %a 939 %2 = load <64 x i8>, <64 x i8>* %b 940 %3 = zext <64 x i8> %1 to <64 x i32> 941 %4 = zext <64 x i8> %2 to <64 x i32> 942 %5 = add nuw nsw <64 x i32> %4, %4 943 %6 = add nuw nsw <64 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 944 %7 = lshr <64 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 945 %8 = trunc <64 x i32> %7 to <64 x i8> 946 store <64 x i8> %8, <64 x i8>* undef, align 4 947 ret void 948} 949 950 951define void @avg_v4i16_2(<4 x i16>* %a, <4 x i16>* %b) nounwind { 952; SSE2-LABEL: avg_v4i16_2: 953; SSE2: # %bb.0: 954; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero 955; SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero 956; SSE2-NEXT: pavgw %xmm0, %xmm1 957; SSE2-NEXT: movq %xmm1, (%rax) 958; SSE2-NEXT: retq 959; 960; AVX-LABEL: avg_v4i16_2: 961; AVX: # %bb.0: 962; AVX-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 963; AVX-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero 964; AVX-NEXT: vpavgw %xmm1, %xmm0, %xmm0 965; AVX-NEXT: vmovq %xmm0, (%rax) 966; AVX-NEXT: retq 967 %1 = load <4 x i16>, <4 x i16>* %a 968 %2 = load <4 x i16>, <4 x i16>* %b 969 %3 = zext <4 x i16> %1 to <4 x i32> 970 %4 = zext <4 x i16> %2 to <4 x i32> 971 %5 = add nuw nsw <4 x i32> %3, %4 972 %6 = add nuw nsw <4 x i32> %5, <i32 1, i32 1, i32 1, i32 1> 973 %7 = lshr <4 x i32> %6, <i32 1, i32 1, i32 1, i32 1> 974 %8 = trunc <4 x i32> %7 to <4 x i16> 975 store <4 x i16> %8, <4 x i16>* undef, align 4 976 ret void 977} 978 979define void @avg_v8i16_2(<8 x i16>* %a, <8 x i16>* %b) nounwind { 980; SSE2-LABEL: avg_v8i16_2: 981; SSE2: # %bb.0: 982; SSE2-NEXT: movdqa (%rdi), %xmm0 983; SSE2-NEXT: pavgw (%rsi), %xmm0 984; SSE2-NEXT: movdqu %xmm0, (%rax) 985; SSE2-NEXT: retq 986; 987; AVX-LABEL: avg_v8i16_2: 988; AVX: # %bb.0: 989; AVX-NEXT: vmovdqa (%rdi), %xmm0 990; AVX-NEXT: vpavgw (%rsi), %xmm0, %xmm0 991; AVX-NEXT: vmovdqu %xmm0, (%rax) 992; AVX-NEXT: retq 993 %1 = load <8 x i16>, <8 x i16>* %a 994 %2 = load <8 x i16>, <8 x i16>* %b 995 %3 = zext <8 x i16> %1 to <8 x i32> 996 %4 = zext <8 x i16> %2 to <8 x i32> 997 %5 = add nuw nsw <8 x i32> %3, %4 998 %6 = add nuw nsw <8 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 999 %7 = lshr <8 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1000 %8 = trunc <8 x i32> %7 to <8 x i16> 1001 store <8 x i16> %8, <8 x i16>* undef, align 4 1002 ret void 1003} 1004 1005define void @avg_v16i16_2(<16 x i16>* %a, <16 x i16>* %b) nounwind { 1006; SSE2-LABEL: avg_v16i16_2: 1007; SSE2: # %bb.0: 1008; SSE2-NEXT: movdqa (%rdi), %xmm0 1009; SSE2-NEXT: movdqa 16(%rdi), %xmm1 1010; SSE2-NEXT: pavgw (%rsi), %xmm0 1011; SSE2-NEXT: pavgw 16(%rsi), %xmm1 1012; SSE2-NEXT: movdqu %xmm1, (%rax) 1013; SSE2-NEXT: movdqu %xmm0, (%rax) 1014; SSE2-NEXT: retq 1015; 1016; AVX1-LABEL: avg_v16i16_2: 1017; AVX1: # %bb.0: 1018; AVX1-NEXT: vmovdqa (%rdi), %xmm0 1019; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 1020; AVX1-NEXT: vpavgw 16(%rsi), %xmm1, %xmm1 1021; AVX1-NEXT: vpavgw (%rsi), %xmm0, %xmm0 1022; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 1023; AVX1-NEXT: vmovups %ymm0, (%rax) 1024; AVX1-NEXT: vzeroupper 1025; AVX1-NEXT: retq 1026; 1027; AVX2-LABEL: avg_v16i16_2: 1028; AVX2: # %bb.0: 1029; AVX2-NEXT: vmovdqa (%rdi), %ymm0 1030; AVX2-NEXT: vpavgw (%rsi), %ymm0, %ymm0 1031; AVX2-NEXT: vmovdqu %ymm0, (%rax) 1032; AVX2-NEXT: vzeroupper 1033; AVX2-NEXT: retq 1034; 1035; AVX512-LABEL: avg_v16i16_2: 1036; AVX512: # %bb.0: 1037; AVX512-NEXT: vmovdqa (%rdi), %ymm0 1038; AVX512-NEXT: vpavgw (%rsi), %ymm0, %ymm0 1039; AVX512-NEXT: vmovdqu %ymm0, (%rax) 1040; AVX512-NEXT: vzeroupper 1041; AVX512-NEXT: retq 1042 %1 = load <16 x i16>, <16 x i16>* %a 1043 %2 = load <16 x i16>, <16 x i16>* %b 1044 %3 = zext <16 x i16> %1 to <16 x i32> 1045 %4 = zext <16 x i16> %2 to <16 x i32> 1046 %5 = add nuw nsw <16 x i32> %3, %4 1047 %6 = add nuw nsw <16 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1048 %7 = lshr <16 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1049 %8 = trunc <16 x i32> %7 to <16 x i16> 1050 store <16 x i16> %8, <16 x i16>* undef, align 4 1051 ret void 1052} 1053 1054define void @avg_v32i16_2(<32 x i16>* %a, <32 x i16>* %b) nounwind { 1055; SSE2-LABEL: avg_v32i16_2: 1056; SSE2: # %bb.0: 1057; SSE2-NEXT: movdqa (%rdi), %xmm0 1058; SSE2-NEXT: movdqa 16(%rdi), %xmm1 1059; SSE2-NEXT: movdqa 32(%rdi), %xmm2 1060; SSE2-NEXT: movdqa 48(%rdi), %xmm3 1061; SSE2-NEXT: pavgw (%rsi), %xmm0 1062; SSE2-NEXT: pavgw 16(%rsi), %xmm1 1063; SSE2-NEXT: pavgw 32(%rsi), %xmm2 1064; SSE2-NEXT: pavgw 48(%rsi), %xmm3 1065; SSE2-NEXT: movdqu %xmm3, (%rax) 1066; SSE2-NEXT: movdqu %xmm2, (%rax) 1067; SSE2-NEXT: movdqu %xmm1, (%rax) 1068; SSE2-NEXT: movdqu %xmm0, (%rax) 1069; SSE2-NEXT: retq 1070; 1071; AVX1-LABEL: avg_v32i16_2: 1072; AVX1: # %bb.0: 1073; AVX1-NEXT: vmovdqa (%rdi), %xmm0 1074; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 1075; AVX1-NEXT: vmovdqa 32(%rdi), %xmm2 1076; AVX1-NEXT: vmovdqa 48(%rdi), %xmm3 1077; AVX1-NEXT: vpavgw 16(%rsi), %xmm1, %xmm1 1078; AVX1-NEXT: vpavgw (%rsi), %xmm0, %xmm0 1079; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 1080; AVX1-NEXT: vpavgw 48(%rsi), %xmm3, %xmm1 1081; AVX1-NEXT: vpavgw 32(%rsi), %xmm2, %xmm2 1082; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1 1083; AVX1-NEXT: vmovups %ymm1, (%rax) 1084; AVX1-NEXT: vmovups %ymm0, (%rax) 1085; AVX1-NEXT: vzeroupper 1086; AVX1-NEXT: retq 1087; 1088; AVX2-LABEL: avg_v32i16_2: 1089; AVX2: # %bb.0: 1090; AVX2-NEXT: vmovdqa (%rdi), %ymm0 1091; AVX2-NEXT: vmovdqa 32(%rdi), %ymm1 1092; AVX2-NEXT: vpavgw (%rsi), %ymm0, %ymm0 1093; AVX2-NEXT: vpavgw 32(%rsi), %ymm1, %ymm1 1094; AVX2-NEXT: vmovdqu %ymm1, (%rax) 1095; AVX2-NEXT: vmovdqu %ymm0, (%rax) 1096; AVX2-NEXT: vzeroupper 1097; AVX2-NEXT: retq 1098; 1099; AVX512F-LABEL: avg_v32i16_2: 1100; AVX512F: # %bb.0: 1101; AVX512F-NEXT: vmovdqa (%rdi), %ymm0 1102; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm1 1103; AVX512F-NEXT: vpavgw (%rsi), %ymm0, %ymm0 1104; AVX512F-NEXT: vpavgw 32(%rsi), %ymm1, %ymm1 1105; AVX512F-NEXT: vmovdqu %ymm1, (%rax) 1106; AVX512F-NEXT: vmovdqu %ymm0, (%rax) 1107; AVX512F-NEXT: vzeroupper 1108; AVX512F-NEXT: retq 1109; 1110; AVX512BW-LABEL: avg_v32i16_2: 1111; AVX512BW: # %bb.0: 1112; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0 1113; AVX512BW-NEXT: vpavgw (%rsi), %zmm0, %zmm0 1114; AVX512BW-NEXT: vmovdqu64 %zmm0, (%rax) 1115; AVX512BW-NEXT: vzeroupper 1116; AVX512BW-NEXT: retq 1117 %1 = load <32 x i16>, <32 x i16>* %a 1118 %2 = load <32 x i16>, <32 x i16>* %b 1119 %3 = zext <32 x i16> %1 to <32 x i32> 1120 %4 = zext <32 x i16> %2 to <32 x i32> 1121 %5 = add nuw nsw <32 x i32> %3, %4 1122 %6 = add nuw nsw <32 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1123 %7 = lshr <32 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1124 %8 = trunc <32 x i32> %7 to <32 x i16> 1125 store <32 x i16> %8, <32 x i16>* undef, align 4 1126 ret void 1127} 1128 1129define void @avg_v4i8_const(<4 x i8>* %a) nounwind { 1130; SSE2-LABEL: avg_v4i8_const: 1131; SSE2: # %bb.0: 1132; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero 1133; SSE2-NEXT: pavgb {{.*}}(%rip), %xmm0 1134; SSE2-NEXT: movd %xmm0, (%rax) 1135; SSE2-NEXT: retq 1136; 1137; AVX-LABEL: avg_v4i8_const: 1138; AVX: # %bb.0: 1139; AVX-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero 1140; AVX-NEXT: vpavgb {{.*}}(%rip), %xmm0, %xmm0 1141; AVX-NEXT: vmovd %xmm0, (%rax) 1142; AVX-NEXT: retq 1143 %1 = load <4 x i8>, <4 x i8>* %a 1144 %2 = zext <4 x i8> %1 to <4 x i32> 1145 %3 = add nuw nsw <4 x i32> %2, <i32 1, i32 2, i32 3, i32 4> 1146 %4 = lshr <4 x i32> %3, <i32 1, i32 1, i32 1, i32 1> 1147 %5 = trunc <4 x i32> %4 to <4 x i8> 1148 store <4 x i8> %5, <4 x i8>* undef, align 4 1149 ret void 1150} 1151 1152define void @avg_v8i8_const(<8 x i8>* %a) nounwind { 1153; SSE2-LABEL: avg_v8i8_const: 1154; SSE2: # %bb.0: 1155; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero 1156; SSE2-NEXT: pavgb {{.*}}(%rip), %xmm0 1157; SSE2-NEXT: movq %xmm0, (%rax) 1158; SSE2-NEXT: retq 1159; 1160; AVX-LABEL: avg_v8i8_const: 1161; AVX: # %bb.0: 1162; AVX-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 1163; AVX-NEXT: vpavgb {{.*}}(%rip), %xmm0, %xmm0 1164; AVX-NEXT: vmovq %xmm0, (%rax) 1165; AVX-NEXT: retq 1166 %1 = load <8 x i8>, <8 x i8>* %a 1167 %2 = zext <8 x i8> %1 to <8 x i32> 1168 %3 = add nuw nsw <8 x i32> %2, <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8> 1169 %4 = lshr <8 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1170 %5 = trunc <8 x i32> %4 to <8 x i8> 1171 store <8 x i8> %5, <8 x i8>* undef, align 4 1172 ret void 1173} 1174 1175define void @avg_v16i8_const(<16 x i8>* %a) nounwind { 1176; SSE2-LABEL: avg_v16i8_const: 1177; SSE2: # %bb.0: 1178; SSE2-NEXT: movdqa (%rdi), %xmm0 1179; SSE2-NEXT: pavgb {{.*}}(%rip), %xmm0 1180; SSE2-NEXT: movdqu %xmm0, (%rax) 1181; SSE2-NEXT: retq 1182; 1183; AVX-LABEL: avg_v16i8_const: 1184; AVX: # %bb.0: 1185; AVX-NEXT: vmovdqa (%rdi), %xmm0 1186; AVX-NEXT: vpavgb {{.*}}(%rip), %xmm0, %xmm0 1187; AVX-NEXT: vmovdqu %xmm0, (%rax) 1188; AVX-NEXT: retq 1189 %1 = load <16 x i8>, <16 x i8>* %a 1190 %2 = zext <16 x i8> %1 to <16 x i32> 1191 %3 = add nuw nsw <16 x i32> %2, <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8> 1192 %4 = lshr <16 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1193 %5 = trunc <16 x i32> %4 to <16 x i8> 1194 store <16 x i8> %5, <16 x i8>* undef, align 4 1195 ret void 1196} 1197 1198define void @avg_v32i8_const(<32 x i8>* %a) nounwind { 1199; SSE2-LABEL: avg_v32i8_const: 1200; SSE2: # %bb.0: 1201; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7] 1202; SSE2-NEXT: movdqa (%rdi), %xmm1 1203; SSE2-NEXT: pavgb %xmm0, %xmm1 1204; SSE2-NEXT: pavgb 16(%rdi), %xmm0 1205; SSE2-NEXT: movdqu %xmm0, (%rax) 1206; SSE2-NEXT: movdqu %xmm1, (%rax) 1207; SSE2-NEXT: retq 1208; 1209; AVX1-LABEL: avg_v32i8_const: 1210; AVX1: # %bb.0: 1211; AVX1-NEXT: vmovddup {{.*#+}} xmm0 = [7.9499288951273625E-275,7.9499288951273625E-275] 1212; AVX1-NEXT: # xmm0 = mem[0,0] 1213; AVX1-NEXT: vpavgb 16(%rdi), %xmm0, %xmm1 1214; AVX1-NEXT: vpavgb (%rdi), %xmm0, %xmm0 1215; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 1216; AVX1-NEXT: vmovups %ymm0, (%rax) 1217; AVX1-NEXT: vzeroupper 1218; AVX1-NEXT: retq 1219; 1220; AVX2-LABEL: avg_v32i8_const: 1221; AVX2: # %bb.0: 1222; AVX2-NEXT: vmovdqa (%rdi), %ymm0 1223; AVX2-NEXT: vpavgb {{.*}}(%rip), %ymm0, %ymm0 1224; AVX2-NEXT: vmovdqu %ymm0, (%rax) 1225; AVX2-NEXT: vzeroupper 1226; AVX2-NEXT: retq 1227; 1228; AVX512-LABEL: avg_v32i8_const: 1229; AVX512: # %bb.0: 1230; AVX512-NEXT: vmovdqa (%rdi), %ymm0 1231; AVX512-NEXT: vpavgb {{.*}}(%rip), %ymm0, %ymm0 1232; AVX512-NEXT: vmovdqu %ymm0, (%rax) 1233; AVX512-NEXT: vzeroupper 1234; AVX512-NEXT: retq 1235 %1 = load <32 x i8>, <32 x i8>* %a 1236 %2 = zext <32 x i8> %1 to <32 x i32> 1237 %3 = add nuw nsw <32 x i32> %2, <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8> 1238 %4 = lshr <32 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1239 %5 = trunc <32 x i32> %4 to <32 x i8> 1240 store <32 x i8> %5, <32 x i8>* undef, align 4 1241 ret void 1242} 1243 1244define void @avg_v64i8_const(<64 x i8>* %a) nounwind { 1245; SSE2-LABEL: avg_v64i8_const: 1246; SSE2: # %bb.0: 1247; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7] 1248; SSE2-NEXT: movdqa (%rdi), %xmm1 1249; SSE2-NEXT: pavgb %xmm0, %xmm1 1250; SSE2-NEXT: movdqa 16(%rdi), %xmm2 1251; SSE2-NEXT: pavgb %xmm0, %xmm2 1252; SSE2-NEXT: movdqa 32(%rdi), %xmm3 1253; SSE2-NEXT: pavgb %xmm0, %xmm3 1254; SSE2-NEXT: pavgb 48(%rdi), %xmm0 1255; SSE2-NEXT: movdqu %xmm0, (%rax) 1256; SSE2-NEXT: movdqu %xmm3, (%rax) 1257; SSE2-NEXT: movdqu %xmm2, (%rax) 1258; SSE2-NEXT: movdqu %xmm1, (%rax) 1259; SSE2-NEXT: retq 1260; 1261; AVX1-LABEL: avg_v64i8_const: 1262; AVX1: # %bb.0: 1263; AVX1-NEXT: vmovddup {{.*#+}} xmm0 = [7.9499288951273625E-275,7.9499288951273625E-275] 1264; AVX1-NEXT: # xmm0 = mem[0,0] 1265; AVX1-NEXT: vpavgb 16(%rdi), %xmm0, %xmm1 1266; AVX1-NEXT: vpavgb (%rdi), %xmm0, %xmm2 1267; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1 1268; AVX1-NEXT: vpavgb 48(%rdi), %xmm0, %xmm2 1269; AVX1-NEXT: vpavgb 32(%rdi), %xmm0, %xmm0 1270; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 1271; AVX1-NEXT: vmovups %ymm0, (%rax) 1272; AVX1-NEXT: vmovups %ymm1, (%rax) 1273; AVX1-NEXT: vzeroupper 1274; AVX1-NEXT: retq 1275; 1276; AVX2-LABEL: avg_v64i8_const: 1277; AVX2: # %bb.0: 1278; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm0 = [506097522914230528,506097522914230528,506097522914230528,506097522914230528] 1279; AVX2-NEXT: vpavgb (%rdi), %ymm0, %ymm1 1280; AVX2-NEXT: vpavgb 32(%rdi), %ymm0, %ymm0 1281; AVX2-NEXT: vmovdqu %ymm0, (%rax) 1282; AVX2-NEXT: vmovdqu %ymm1, (%rax) 1283; AVX2-NEXT: vzeroupper 1284; AVX2-NEXT: retq 1285; 1286; AVX512F-LABEL: avg_v64i8_const: 1287; AVX512F: # %bb.0: 1288; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm0 = [506097522914230528,506097522914230528,506097522914230528,506097522914230528] 1289; AVX512F-NEXT: vpavgb (%rdi), %ymm0, %ymm1 1290; AVX512F-NEXT: vpavgb 32(%rdi), %ymm0, %ymm0 1291; AVX512F-NEXT: vmovdqu %ymm0, (%rax) 1292; AVX512F-NEXT: vmovdqu %ymm1, (%rax) 1293; AVX512F-NEXT: vzeroupper 1294; AVX512F-NEXT: retq 1295; 1296; AVX512BW-LABEL: avg_v64i8_const: 1297; AVX512BW: # %bb.0: 1298; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0 1299; AVX512BW-NEXT: vpavgb {{.*}}(%rip), %zmm0, %zmm0 1300; AVX512BW-NEXT: vmovdqu64 %zmm0, (%rax) 1301; AVX512BW-NEXT: vzeroupper 1302; AVX512BW-NEXT: retq 1303 %1 = load <64 x i8>, <64 x i8>* %a 1304 %2 = zext <64 x i8> %1 to <64 x i32> 1305 %3 = add nuw nsw <64 x i32> %2, <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8> 1306 %4 = lshr <64 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1307 %5 = trunc <64 x i32> %4 to <64 x i8> 1308 store <64 x i8> %5, <64 x i8>* undef, align 4 1309 ret void 1310} 1311 1312define void @avg_v4i16_const(<4 x i16>* %a) nounwind { 1313; SSE2-LABEL: avg_v4i16_const: 1314; SSE2: # %bb.0: 1315; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero 1316; SSE2-NEXT: pavgw {{.*}}(%rip), %xmm0 1317; SSE2-NEXT: movq %xmm0, (%rax) 1318; SSE2-NEXT: retq 1319; 1320; AVX-LABEL: avg_v4i16_const: 1321; AVX: # %bb.0: 1322; AVX-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 1323; AVX-NEXT: vpavgw {{.*}}(%rip), %xmm0, %xmm0 1324; AVX-NEXT: vmovq %xmm0, (%rax) 1325; AVX-NEXT: retq 1326 %1 = load <4 x i16>, <4 x i16>* %a 1327 %2 = zext <4 x i16> %1 to <4 x i32> 1328 %3 = add nuw nsw <4 x i32> %2, <i32 1, i32 2, i32 3, i32 4> 1329 %4 = lshr <4 x i32> %3, <i32 1, i32 1, i32 1, i32 1> 1330 %5 = trunc <4 x i32> %4 to <4 x i16> 1331 store <4 x i16> %5, <4 x i16>* undef, align 4 1332 ret void 1333} 1334 1335define void @avg_v8i16_const(<8 x i16>* %a) nounwind { 1336; SSE2-LABEL: avg_v8i16_const: 1337; SSE2: # %bb.0: 1338; SSE2-NEXT: movdqa (%rdi), %xmm0 1339; SSE2-NEXT: pavgw {{.*}}(%rip), %xmm0 1340; SSE2-NEXT: movdqu %xmm0, (%rax) 1341; SSE2-NEXT: retq 1342; 1343; AVX-LABEL: avg_v8i16_const: 1344; AVX: # %bb.0: 1345; AVX-NEXT: vmovdqa (%rdi), %xmm0 1346; AVX-NEXT: vpavgw {{.*}}(%rip), %xmm0, %xmm0 1347; AVX-NEXT: vmovdqu %xmm0, (%rax) 1348; AVX-NEXT: retq 1349 %1 = load <8 x i16>, <8 x i16>* %a 1350 %2 = zext <8 x i16> %1 to <8 x i32> 1351 %3 = add nuw nsw <8 x i32> %2, <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8> 1352 %4 = lshr <8 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1353 %5 = trunc <8 x i32> %4 to <8 x i16> 1354 store <8 x i16> %5, <8 x i16>* undef, align 4 1355 ret void 1356} 1357 1358define void @avg_v16i16_const(<16 x i16>* %a) nounwind { 1359; SSE2-LABEL: avg_v16i16_const: 1360; SSE2: # %bb.0: 1361; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [0,1,2,3,4,5,6,7] 1362; SSE2-NEXT: movdqa (%rdi), %xmm1 1363; SSE2-NEXT: pavgw %xmm0, %xmm1 1364; SSE2-NEXT: pavgw 16(%rdi), %xmm0 1365; SSE2-NEXT: movdqu %xmm0, (%rax) 1366; SSE2-NEXT: movdqu %xmm1, (%rax) 1367; SSE2-NEXT: retq 1368; 1369; AVX1-LABEL: avg_v16i16_const: 1370; AVX1: # %bb.0: 1371; AVX1-NEXT: vmovdqa {{.*#+}} xmm0 = [0,1,2,3,4,5,6,7] 1372; AVX1-NEXT: vpavgw 16(%rdi), %xmm0, %xmm1 1373; AVX1-NEXT: vpavgw (%rdi), %xmm0, %xmm0 1374; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 1375; AVX1-NEXT: vmovups %ymm0, (%rax) 1376; AVX1-NEXT: vzeroupper 1377; AVX1-NEXT: retq 1378; 1379; AVX2-LABEL: avg_v16i16_const: 1380; AVX2: # %bb.0: 1381; AVX2-NEXT: vmovdqa (%rdi), %ymm0 1382; AVX2-NEXT: vpavgw {{.*}}(%rip), %ymm0, %ymm0 1383; AVX2-NEXT: vmovdqu %ymm0, (%rax) 1384; AVX2-NEXT: vzeroupper 1385; AVX2-NEXT: retq 1386; 1387; AVX512-LABEL: avg_v16i16_const: 1388; AVX512: # %bb.0: 1389; AVX512-NEXT: vmovdqa (%rdi), %ymm0 1390; AVX512-NEXT: vpavgw {{.*}}(%rip), %ymm0, %ymm0 1391; AVX512-NEXT: vmovdqu %ymm0, (%rax) 1392; AVX512-NEXT: vzeroupper 1393; AVX512-NEXT: retq 1394 %1 = load <16 x i16>, <16 x i16>* %a 1395 %2 = zext <16 x i16> %1 to <16 x i32> 1396 %3 = add nuw nsw <16 x i32> %2, <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8> 1397 %4 = lshr <16 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1398 %5 = trunc <16 x i32> %4 to <16 x i16> 1399 store <16 x i16> %5, <16 x i16>* undef, align 4 1400 ret void 1401} 1402 1403define void @avg_v32i16_const(<32 x i16>* %a) nounwind { 1404; SSE2-LABEL: avg_v32i16_const: 1405; SSE2: # %bb.0: 1406; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [0,1,2,3,4,5,6,7] 1407; SSE2-NEXT: movdqa (%rdi), %xmm1 1408; SSE2-NEXT: pavgw %xmm0, %xmm1 1409; SSE2-NEXT: movdqa 16(%rdi), %xmm2 1410; SSE2-NEXT: pavgw %xmm0, %xmm2 1411; SSE2-NEXT: movdqa 32(%rdi), %xmm3 1412; SSE2-NEXT: pavgw %xmm0, %xmm3 1413; SSE2-NEXT: pavgw 48(%rdi), %xmm0 1414; SSE2-NEXT: movdqu %xmm0, (%rax) 1415; SSE2-NEXT: movdqu %xmm3, (%rax) 1416; SSE2-NEXT: movdqu %xmm2, (%rax) 1417; SSE2-NEXT: movdqu %xmm1, (%rax) 1418; SSE2-NEXT: retq 1419; 1420; AVX1-LABEL: avg_v32i16_const: 1421; AVX1: # %bb.0: 1422; AVX1-NEXT: vmovdqa {{.*#+}} xmm0 = [0,1,2,3,4,5,6,7] 1423; AVX1-NEXT: vpavgw 16(%rdi), %xmm0, %xmm1 1424; AVX1-NEXT: vpavgw (%rdi), %xmm0, %xmm2 1425; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1 1426; AVX1-NEXT: vpavgw 48(%rdi), %xmm0, %xmm2 1427; AVX1-NEXT: vpavgw 32(%rdi), %xmm0, %xmm0 1428; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 1429; AVX1-NEXT: vmovups %ymm0, (%rax) 1430; AVX1-NEXT: vmovups %ymm1, (%rax) 1431; AVX1-NEXT: vzeroupper 1432; AVX1-NEXT: retq 1433; 1434; AVX2-LABEL: avg_v32i16_const: 1435; AVX2: # %bb.0: 1436; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7] 1437; AVX2-NEXT: # ymm0 = mem[0,1,0,1] 1438; AVX2-NEXT: vpavgw (%rdi), %ymm0, %ymm1 1439; AVX2-NEXT: vpavgw 32(%rdi), %ymm0, %ymm0 1440; AVX2-NEXT: vmovdqu %ymm0, (%rax) 1441; AVX2-NEXT: vmovdqu %ymm1, (%rax) 1442; AVX2-NEXT: vzeroupper 1443; AVX2-NEXT: retq 1444; 1445; AVX512F-LABEL: avg_v32i16_const: 1446; AVX512F: # %bb.0: 1447; AVX512F-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7] 1448; AVX512F-NEXT: # ymm0 = mem[0,1,0,1] 1449; AVX512F-NEXT: vpavgw (%rdi), %ymm0, %ymm1 1450; AVX512F-NEXT: vpavgw 32(%rdi), %ymm0, %ymm0 1451; AVX512F-NEXT: vmovdqu %ymm0, (%rax) 1452; AVX512F-NEXT: vmovdqu %ymm1, (%rax) 1453; AVX512F-NEXT: vzeroupper 1454; AVX512F-NEXT: retq 1455; 1456; AVX512BW-LABEL: avg_v32i16_const: 1457; AVX512BW: # %bb.0: 1458; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0 1459; AVX512BW-NEXT: vpavgw {{.*}}(%rip), %zmm0, %zmm0 1460; AVX512BW-NEXT: vmovdqu64 %zmm0, (%rax) 1461; AVX512BW-NEXT: vzeroupper 1462; AVX512BW-NEXT: retq 1463 %1 = load <32 x i16>, <32 x i16>* %a 1464 %2 = zext <32 x i16> %1 to <32 x i32> 1465 %3 = add nuw nsw <32 x i32> %2, <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8> 1466 %4 = lshr <32 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1467 %5 = trunc <32 x i32> %4 to <32 x i16> 1468 store <32 x i16> %5, <32 x i16>* undef, align 4 1469 ret void 1470} 1471 1472define <16 x i8> @avg_v16i8_3(<16 x i8> %a, <16 x i8> %b) nounwind { 1473; SSE2-LABEL: avg_v16i8_3: 1474; SSE2: # %bb.0: 1475; SSE2-NEXT: pavgb %xmm1, %xmm0 1476; SSE2-NEXT: retq 1477; 1478; AVX-LABEL: avg_v16i8_3: 1479; AVX: # %bb.0: 1480; AVX-NEXT: vpavgb %xmm1, %xmm0, %xmm0 1481; AVX-NEXT: retq 1482 %za = zext <16 x i8> %a to <16 x i16> 1483 %zb = zext <16 x i8> %b to <16 x i16> 1484 %add = add nuw nsw <16 x i16> %za, %zb 1485 %add1 = add nuw nsw <16 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> 1486 %lshr = lshr <16 x i16> %add1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> 1487 %res = trunc <16 x i16> %lshr to <16 x i8> 1488 ret <16 x i8> %res 1489} 1490 1491define <32 x i8> @avg_v32i8_3(<32 x i8> %a, <32 x i8> %b) nounwind { 1492; SSE2-LABEL: avg_v32i8_3: 1493; SSE2: # %bb.0: 1494; SSE2-NEXT: pavgb %xmm2, %xmm0 1495; SSE2-NEXT: pavgb %xmm3, %xmm1 1496; SSE2-NEXT: retq 1497; 1498; AVX1-LABEL: avg_v32i8_3: 1499; AVX1: # %bb.0: 1500; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 1501; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 1502; AVX1-NEXT: vpavgb %xmm2, %xmm3, %xmm2 1503; AVX1-NEXT: vpavgb %xmm1, %xmm0, %xmm0 1504; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 1505; AVX1-NEXT: retq 1506; 1507; AVX2-LABEL: avg_v32i8_3: 1508; AVX2: # %bb.0: 1509; AVX2-NEXT: vpavgb %ymm1, %ymm0, %ymm0 1510; AVX2-NEXT: retq 1511; 1512; AVX512-LABEL: avg_v32i8_3: 1513; AVX512: # %bb.0: 1514; AVX512-NEXT: vpavgb %ymm1, %ymm0, %ymm0 1515; AVX512-NEXT: retq 1516 %za = zext <32 x i8> %a to <32 x i16> 1517 %zb = zext <32 x i8> %b to <32 x i16> 1518 %add = add nuw nsw <32 x i16> %za, %zb 1519 %add1 = add nuw nsw <32 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> 1520 %lshr = lshr <32 x i16> %add1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> 1521 %res = trunc <32 x i16> %lshr to <32 x i8> 1522 ret <32 x i8> %res 1523} 1524 1525define <64 x i8> @avg_v64i8_3(<64 x i8> %a, <64 x i8> %b) nounwind { 1526; SSE2-LABEL: avg_v64i8_3: 1527; SSE2: # %bb.0: 1528; SSE2-NEXT: pavgb %xmm4, %xmm0 1529; SSE2-NEXT: pavgb %xmm5, %xmm1 1530; SSE2-NEXT: pavgb %xmm6, %xmm2 1531; SSE2-NEXT: pavgb %xmm7, %xmm3 1532; SSE2-NEXT: retq 1533; 1534; AVX1-LABEL: avg_v64i8_3: 1535; AVX1: # %bb.0: 1536; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 1537; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 1538; AVX1-NEXT: vpavgb %xmm4, %xmm5, %xmm4 1539; AVX1-NEXT: vpavgb %xmm2, %xmm0, %xmm0 1540; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 1541; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 1542; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 1543; AVX1-NEXT: vpavgb %xmm2, %xmm4, %xmm2 1544; AVX1-NEXT: vpavgb %xmm3, %xmm1, %xmm1 1545; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 1546; AVX1-NEXT: retq 1547; 1548; AVX2-LABEL: avg_v64i8_3: 1549; AVX2: # %bb.0: 1550; AVX2-NEXT: vpavgb %ymm2, %ymm0, %ymm0 1551; AVX2-NEXT: vpavgb %ymm3, %ymm1, %ymm1 1552; AVX2-NEXT: retq 1553; 1554; AVX512F-LABEL: avg_v64i8_3: 1555; AVX512F: # %bb.0: 1556; AVX512F-NEXT: vpavgb %ymm2, %ymm0, %ymm0 1557; AVX512F-NEXT: vpavgb %ymm3, %ymm1, %ymm1 1558; AVX512F-NEXT: retq 1559; 1560; AVX512BW-LABEL: avg_v64i8_3: 1561; AVX512BW: # %bb.0: 1562; AVX512BW-NEXT: vpavgb %zmm1, %zmm0, %zmm0 1563; AVX512BW-NEXT: retq 1564 %za = zext <64 x i8> %a to <64 x i16> 1565 %zb = zext <64 x i8> %b to <64 x i16> 1566 %add = add nuw nsw <64 x i16> %za, %zb 1567 %add1 = add nuw nsw <64 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> 1568 %lshr = lshr <64 x i16> %add1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> 1569 %res = trunc <64 x i16> %lshr to <64 x i8> 1570 ret <64 x i8> %res 1571} 1572 1573define <512 x i8> @avg_v512i8_3(<512 x i8> %a, <512 x i8> %b) nounwind { 1574; SSE2-LABEL: avg_v512i8_3: 1575; SSE2: # %bb.0: 1576; SSE2-NEXT: movq %rdi, %rax 1577; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1578; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1579; SSE2-NEXT: movdqa %xmm8, 496(%rdi) 1580; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1581; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1582; SSE2-NEXT: movdqa %xmm8, 480(%rdi) 1583; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1584; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1585; SSE2-NEXT: movdqa %xmm8, 464(%rdi) 1586; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1587; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1588; SSE2-NEXT: movdqa %xmm8, 448(%rdi) 1589; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1590; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1591; SSE2-NEXT: movdqa %xmm8, 432(%rdi) 1592; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1593; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1594; SSE2-NEXT: movdqa %xmm8, 416(%rdi) 1595; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1596; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1597; SSE2-NEXT: movdqa %xmm8, 400(%rdi) 1598; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1599; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1600; SSE2-NEXT: movdqa %xmm8, 384(%rdi) 1601; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1602; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1603; SSE2-NEXT: movdqa %xmm8, 368(%rdi) 1604; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1605; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1606; SSE2-NEXT: movdqa %xmm8, 352(%rdi) 1607; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1608; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1609; SSE2-NEXT: movdqa %xmm8, 336(%rdi) 1610; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1611; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1612; SSE2-NEXT: movdqa %xmm8, 320(%rdi) 1613; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1614; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1615; SSE2-NEXT: movdqa %xmm8, 304(%rdi) 1616; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1617; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1618; SSE2-NEXT: movdqa %xmm8, 288(%rdi) 1619; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1620; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1621; SSE2-NEXT: movdqa %xmm8, 272(%rdi) 1622; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1623; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1624; SSE2-NEXT: movdqa %xmm8, 256(%rdi) 1625; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1626; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1627; SSE2-NEXT: movdqa %xmm8, 240(%rdi) 1628; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1629; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1630; SSE2-NEXT: movdqa %xmm8, 224(%rdi) 1631; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1632; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1633; SSE2-NEXT: movdqa %xmm8, 208(%rdi) 1634; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1635; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1636; SSE2-NEXT: movdqa %xmm8, 192(%rdi) 1637; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1638; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1639; SSE2-NEXT: movdqa %xmm8, 176(%rdi) 1640; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1641; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1642; SSE2-NEXT: movdqa %xmm8, 160(%rdi) 1643; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1644; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1645; SSE2-NEXT: movdqa %xmm8, 144(%rdi) 1646; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1647; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1648; SSE2-NEXT: movdqa %xmm8, 128(%rdi) 1649; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm7 1650; SSE2-NEXT: movdqa %xmm7, 112(%rdi) 1651; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm6 1652; SSE2-NEXT: movdqa %xmm6, 96(%rdi) 1653; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm5 1654; SSE2-NEXT: movdqa %xmm5, 80(%rdi) 1655; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm4 1656; SSE2-NEXT: movdqa %xmm4, 64(%rdi) 1657; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm3 1658; SSE2-NEXT: movdqa %xmm3, 48(%rdi) 1659; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm2 1660; SSE2-NEXT: movdqa %xmm2, 32(%rdi) 1661; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm1 1662; SSE2-NEXT: movdqa %xmm1, 16(%rdi) 1663; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm0 1664; SSE2-NEXT: movdqa %xmm0, (%rdi) 1665; SSE2-NEXT: retq 1666; 1667; AVX1-LABEL: avg_v512i8_3: 1668; AVX1: # %bb.0: 1669; AVX1-NEXT: pushq %rbp 1670; AVX1-NEXT: movq %rsp, %rbp 1671; AVX1-NEXT: andq $-32, %rsp 1672; AVX1-NEXT: subq $96, %rsp 1673; AVX1-NEXT: movq %rdi, %rax 1674; AVX1-NEXT: vpavgb 272(%rbp), %xmm0, %xmm8 1675; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 1676; AVX1-NEXT: vpavgb 288(%rbp), %xmm0, %xmm0 1677; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm8, %ymm0 1678; AVX1-NEXT: vmovaps %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill 1679; AVX1-NEXT: vpavgb 304(%rbp), %xmm1, %xmm8 1680; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1 1681; AVX1-NEXT: vpavgb 320(%rbp), %xmm1, %xmm1 1682; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm8, %ymm0 1683; AVX1-NEXT: vmovaps %ymm0, (%rsp) # 32-byte Spill 1684; AVX1-NEXT: vpavgb 336(%rbp), %xmm2, %xmm8 1685; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2 1686; AVX1-NEXT: vpavgb 352(%rbp), %xmm2, %xmm2 1687; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm8, %ymm13 1688; AVX1-NEXT: vpavgb 368(%rbp), %xmm3, %xmm8 1689; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm3 1690; AVX1-NEXT: vpavgb 384(%rbp), %xmm3, %xmm3 1691; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm8, %ymm14 1692; AVX1-NEXT: vpavgb 400(%rbp), %xmm4, %xmm8 1693; AVX1-NEXT: vextractf128 $1, %ymm4, %xmm4 1694; AVX1-NEXT: vpavgb 416(%rbp), %xmm4, %xmm4 1695; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm8, %ymm15 1696; AVX1-NEXT: vpavgb 432(%rbp), %xmm5, %xmm8 1697; AVX1-NEXT: vextractf128 $1, %ymm5, %xmm5 1698; AVX1-NEXT: vpavgb 448(%rbp), %xmm5, %xmm5 1699; AVX1-NEXT: vinsertf128 $1, %xmm5, %ymm8, %ymm12 1700; AVX1-NEXT: vpavgb 464(%rbp), %xmm6, %xmm8 1701; AVX1-NEXT: vextractf128 $1, %ymm6, %xmm6 1702; AVX1-NEXT: vpavgb 480(%rbp), %xmm6, %xmm6 1703; AVX1-NEXT: vinsertf128 $1, %xmm6, %ymm8, %ymm6 1704; AVX1-NEXT: vpavgb 496(%rbp), %xmm7, %xmm8 1705; AVX1-NEXT: vextractf128 $1, %ymm7, %xmm7 1706; AVX1-NEXT: vpavgb 512(%rbp), %xmm7, %xmm7 1707; AVX1-NEXT: vinsertf128 $1, %xmm7, %ymm8, %ymm7 1708; AVX1-NEXT: vmovdqa 16(%rbp), %xmm0 1709; AVX1-NEXT: vmovdqa 32(%rbp), %xmm1 1710; AVX1-NEXT: vpavgb 528(%rbp), %xmm0, %xmm0 1711; AVX1-NEXT: vpavgb 544(%rbp), %xmm1, %xmm1 1712; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm8 1713; AVX1-NEXT: vmovdqa 48(%rbp), %xmm0 1714; AVX1-NEXT: vmovdqa 64(%rbp), %xmm1 1715; AVX1-NEXT: vpavgb 560(%rbp), %xmm0, %xmm0 1716; AVX1-NEXT: vpavgb 576(%rbp), %xmm1, %xmm1 1717; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm9 1718; AVX1-NEXT: vmovdqa 80(%rbp), %xmm0 1719; AVX1-NEXT: vmovdqa 96(%rbp), %xmm1 1720; AVX1-NEXT: vpavgb 592(%rbp), %xmm0, %xmm0 1721; AVX1-NEXT: vpavgb 608(%rbp), %xmm1, %xmm1 1722; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm10 1723; AVX1-NEXT: vmovdqa 112(%rbp), %xmm0 1724; AVX1-NEXT: vmovdqa 128(%rbp), %xmm1 1725; AVX1-NEXT: vpavgb 624(%rbp), %xmm0, %xmm0 1726; AVX1-NEXT: vpavgb 640(%rbp), %xmm1, %xmm1 1727; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 1728; AVX1-NEXT: vmovdqa 144(%rbp), %xmm1 1729; AVX1-NEXT: vmovdqa 160(%rbp), %xmm2 1730; AVX1-NEXT: vpavgb 656(%rbp), %xmm1, %xmm1 1731; AVX1-NEXT: vpavgb 672(%rbp), %xmm2, %xmm2 1732; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 1733; AVX1-NEXT: vmovdqa 176(%rbp), %xmm2 1734; AVX1-NEXT: vmovdqa 192(%rbp), %xmm3 1735; AVX1-NEXT: vpavgb 688(%rbp), %xmm2, %xmm2 1736; AVX1-NEXT: vpavgb 704(%rbp), %xmm3, %xmm3 1737; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm2, %ymm2 1738; AVX1-NEXT: vmovdqa 208(%rbp), %xmm3 1739; AVX1-NEXT: vmovdqa 224(%rbp), %xmm4 1740; AVX1-NEXT: vpavgb 720(%rbp), %xmm3, %xmm3 1741; AVX1-NEXT: vpavgb 736(%rbp), %xmm4, %xmm4 1742; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3 1743; AVX1-NEXT: vmovdqa 240(%rbp), %xmm4 1744; AVX1-NEXT: vpavgb 752(%rbp), %xmm4, %xmm4 1745; AVX1-NEXT: vmovdqa 256(%rbp), %xmm11 1746; AVX1-NEXT: vpavgb 768(%rbp), %xmm11, %xmm5 1747; AVX1-NEXT: vinsertf128 $1, %xmm5, %ymm4, %ymm4 1748; AVX1-NEXT: vmovaps %ymm4, 480(%rdi) 1749; AVX1-NEXT: vmovaps %ymm3, 448(%rdi) 1750; AVX1-NEXT: vmovaps %ymm2, 416(%rdi) 1751; AVX1-NEXT: vmovaps %ymm1, 384(%rdi) 1752; AVX1-NEXT: vmovaps %ymm0, 352(%rdi) 1753; AVX1-NEXT: vmovaps %ymm10, 320(%rdi) 1754; AVX1-NEXT: vmovaps %ymm9, 288(%rdi) 1755; AVX1-NEXT: vmovaps %ymm8, 256(%rdi) 1756; AVX1-NEXT: vmovaps %ymm7, 224(%rdi) 1757; AVX1-NEXT: vmovaps %ymm6, 192(%rdi) 1758; AVX1-NEXT: vmovaps %ymm12, 160(%rdi) 1759; AVX1-NEXT: vmovaps %ymm15, 128(%rdi) 1760; AVX1-NEXT: vmovaps %ymm14, 96(%rdi) 1761; AVX1-NEXT: vmovaps %ymm13, 64(%rdi) 1762; AVX1-NEXT: vmovaps (%rsp), %ymm0 # 32-byte Reload 1763; AVX1-NEXT: vmovaps %ymm0, 32(%rdi) 1764; AVX1-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload 1765; AVX1-NEXT: vmovaps %ymm0, (%rdi) 1766; AVX1-NEXT: movq %rbp, %rsp 1767; AVX1-NEXT: popq %rbp 1768; AVX1-NEXT: vzeroupper 1769; AVX1-NEXT: retq 1770; 1771; AVX2-LABEL: avg_v512i8_3: 1772; AVX2: # %bb.0: 1773; AVX2-NEXT: pushq %rbp 1774; AVX2-NEXT: movq %rsp, %rbp 1775; AVX2-NEXT: andq $-32, %rsp 1776; AVX2-NEXT: subq $32, %rsp 1777; AVX2-NEXT: movq %rdi, %rax 1778; AVX2-NEXT: vmovdqa 240(%rbp), %ymm8 1779; AVX2-NEXT: vmovdqa 208(%rbp), %ymm9 1780; AVX2-NEXT: vmovdqa 176(%rbp), %ymm10 1781; AVX2-NEXT: vmovdqa 144(%rbp), %ymm11 1782; AVX2-NEXT: vmovdqa 112(%rbp), %ymm12 1783; AVX2-NEXT: vmovdqa 80(%rbp), %ymm13 1784; AVX2-NEXT: vmovdqa 48(%rbp), %ymm14 1785; AVX2-NEXT: vmovdqa 16(%rbp), %ymm15 1786; AVX2-NEXT: vpavgb 272(%rbp), %ymm0, %ymm0 1787; AVX2-NEXT: vpavgb 304(%rbp), %ymm1, %ymm1 1788; AVX2-NEXT: vpavgb 336(%rbp), %ymm2, %ymm2 1789; AVX2-NEXT: vpavgb 368(%rbp), %ymm3, %ymm3 1790; AVX2-NEXT: vpavgb 400(%rbp), %ymm4, %ymm4 1791; AVX2-NEXT: vpavgb 432(%rbp), %ymm5, %ymm5 1792; AVX2-NEXT: vpavgb 464(%rbp), %ymm6, %ymm6 1793; AVX2-NEXT: vpavgb 496(%rbp), %ymm7, %ymm7 1794; AVX2-NEXT: vpavgb 528(%rbp), %ymm15, %ymm15 1795; AVX2-NEXT: vpavgb 560(%rbp), %ymm14, %ymm14 1796; AVX2-NEXT: vpavgb 592(%rbp), %ymm13, %ymm13 1797; AVX2-NEXT: vpavgb 624(%rbp), %ymm12, %ymm12 1798; AVX2-NEXT: vpavgb 656(%rbp), %ymm11, %ymm11 1799; AVX2-NEXT: vpavgb 688(%rbp), %ymm10, %ymm10 1800; AVX2-NEXT: vpavgb 720(%rbp), %ymm9, %ymm9 1801; AVX2-NEXT: vpavgb 752(%rbp), %ymm8, %ymm8 1802; AVX2-NEXT: vmovdqa %ymm8, 480(%rdi) 1803; AVX2-NEXT: vmovdqa %ymm9, 448(%rdi) 1804; AVX2-NEXT: vmovdqa %ymm10, 416(%rdi) 1805; AVX2-NEXT: vmovdqa %ymm11, 384(%rdi) 1806; AVX2-NEXT: vmovdqa %ymm12, 352(%rdi) 1807; AVX2-NEXT: vmovdqa %ymm13, 320(%rdi) 1808; AVX2-NEXT: vmovdqa %ymm14, 288(%rdi) 1809; AVX2-NEXT: vmovdqa %ymm15, 256(%rdi) 1810; AVX2-NEXT: vmovdqa %ymm7, 224(%rdi) 1811; AVX2-NEXT: vmovdqa %ymm6, 192(%rdi) 1812; AVX2-NEXT: vmovdqa %ymm5, 160(%rdi) 1813; AVX2-NEXT: vmovdqa %ymm4, 128(%rdi) 1814; AVX2-NEXT: vmovdqa %ymm3, 96(%rdi) 1815; AVX2-NEXT: vmovdqa %ymm2, 64(%rdi) 1816; AVX2-NEXT: vmovdqa %ymm1, 32(%rdi) 1817; AVX2-NEXT: vmovdqa %ymm0, (%rdi) 1818; AVX2-NEXT: movq %rbp, %rsp 1819; AVX2-NEXT: popq %rbp 1820; AVX2-NEXT: vzeroupper 1821; AVX2-NEXT: retq 1822; 1823; AVX512F-LABEL: avg_v512i8_3: 1824; AVX512F: # %bb.0: 1825; AVX512F-NEXT: pushq %rbp 1826; AVX512F-NEXT: movq %rsp, %rbp 1827; AVX512F-NEXT: andq $-32, %rsp 1828; AVX512F-NEXT: subq $32, %rsp 1829; AVX512F-NEXT: movq %rdi, %rax 1830; AVX512F-NEXT: vmovdqa 240(%rbp), %ymm8 1831; AVX512F-NEXT: vmovdqa 208(%rbp), %ymm9 1832; AVX512F-NEXT: vmovdqa 176(%rbp), %ymm10 1833; AVX512F-NEXT: vmovdqa 144(%rbp), %ymm11 1834; AVX512F-NEXT: vmovdqa 112(%rbp), %ymm12 1835; AVX512F-NEXT: vmovdqa 80(%rbp), %ymm13 1836; AVX512F-NEXT: vmovdqa 48(%rbp), %ymm14 1837; AVX512F-NEXT: vmovdqa 16(%rbp), %ymm15 1838; AVX512F-NEXT: vpavgb 272(%rbp), %ymm0, %ymm0 1839; AVX512F-NEXT: vpavgb 304(%rbp), %ymm1, %ymm1 1840; AVX512F-NEXT: vpavgb 336(%rbp), %ymm2, %ymm2 1841; AVX512F-NEXT: vpavgb 368(%rbp), %ymm3, %ymm3 1842; AVX512F-NEXT: vpavgb 400(%rbp), %ymm4, %ymm4 1843; AVX512F-NEXT: vpavgb 432(%rbp), %ymm5, %ymm5 1844; AVX512F-NEXT: vpavgb 464(%rbp), %ymm6, %ymm6 1845; AVX512F-NEXT: vpavgb 496(%rbp), %ymm7, %ymm7 1846; AVX512F-NEXT: vpavgb 528(%rbp), %ymm15, %ymm15 1847; AVX512F-NEXT: vpavgb 560(%rbp), %ymm14, %ymm14 1848; AVX512F-NEXT: vpavgb 592(%rbp), %ymm13, %ymm13 1849; AVX512F-NEXT: vpavgb 624(%rbp), %ymm12, %ymm12 1850; AVX512F-NEXT: vpavgb 656(%rbp), %ymm11, %ymm11 1851; AVX512F-NEXT: vpavgb 688(%rbp), %ymm10, %ymm10 1852; AVX512F-NEXT: vpavgb 720(%rbp), %ymm9, %ymm9 1853; AVX512F-NEXT: vpavgb 752(%rbp), %ymm8, %ymm8 1854; AVX512F-NEXT: vmovdqa %ymm8, 480(%rdi) 1855; AVX512F-NEXT: vmovdqa %ymm9, 448(%rdi) 1856; AVX512F-NEXT: vmovdqa %ymm10, 416(%rdi) 1857; AVX512F-NEXT: vmovdqa %ymm11, 384(%rdi) 1858; AVX512F-NEXT: vmovdqa %ymm12, 352(%rdi) 1859; AVX512F-NEXT: vmovdqa %ymm13, 320(%rdi) 1860; AVX512F-NEXT: vmovdqa %ymm14, 288(%rdi) 1861; AVX512F-NEXT: vmovdqa %ymm15, 256(%rdi) 1862; AVX512F-NEXT: vmovdqa %ymm7, 224(%rdi) 1863; AVX512F-NEXT: vmovdqa %ymm6, 192(%rdi) 1864; AVX512F-NEXT: vmovdqa %ymm5, 160(%rdi) 1865; AVX512F-NEXT: vmovdqa %ymm4, 128(%rdi) 1866; AVX512F-NEXT: vmovdqa %ymm3, 96(%rdi) 1867; AVX512F-NEXT: vmovdqa %ymm2, 64(%rdi) 1868; AVX512F-NEXT: vmovdqa %ymm1, 32(%rdi) 1869; AVX512F-NEXT: vmovdqa %ymm0, (%rdi) 1870; AVX512F-NEXT: movq %rbp, %rsp 1871; AVX512F-NEXT: popq %rbp 1872; AVX512F-NEXT: vzeroupper 1873; AVX512F-NEXT: retq 1874; 1875; AVX512BW-LABEL: avg_v512i8_3: 1876; AVX512BW: # %bb.0: 1877; AVX512BW-NEXT: pushq %rbp 1878; AVX512BW-NEXT: movq %rsp, %rbp 1879; AVX512BW-NEXT: andq $-64, %rsp 1880; AVX512BW-NEXT: subq $64, %rsp 1881; AVX512BW-NEXT: movq %rdi, %rax 1882; AVX512BW-NEXT: vpavgb 16(%rbp), %zmm0, %zmm0 1883; AVX512BW-NEXT: vpavgb 80(%rbp), %zmm1, %zmm1 1884; AVX512BW-NEXT: vpavgb 144(%rbp), %zmm2, %zmm2 1885; AVX512BW-NEXT: vpavgb 208(%rbp), %zmm3, %zmm3 1886; AVX512BW-NEXT: vpavgb 272(%rbp), %zmm4, %zmm4 1887; AVX512BW-NEXT: vpavgb 336(%rbp), %zmm5, %zmm5 1888; AVX512BW-NEXT: vpavgb 400(%rbp), %zmm6, %zmm6 1889; AVX512BW-NEXT: vpavgb 464(%rbp), %zmm7, %zmm7 1890; AVX512BW-NEXT: vmovdqa64 %zmm7, 448(%rdi) 1891; AVX512BW-NEXT: vmovdqa64 %zmm6, 384(%rdi) 1892; AVX512BW-NEXT: vmovdqa64 %zmm5, 320(%rdi) 1893; AVX512BW-NEXT: vmovdqa64 %zmm4, 256(%rdi) 1894; AVX512BW-NEXT: vmovdqa64 %zmm3, 192(%rdi) 1895; AVX512BW-NEXT: vmovdqa64 %zmm2, 128(%rdi) 1896; AVX512BW-NEXT: vmovdqa64 %zmm1, 64(%rdi) 1897; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdi) 1898; AVX512BW-NEXT: movq %rbp, %rsp 1899; AVX512BW-NEXT: popq %rbp 1900; AVX512BW-NEXT: vzeroupper 1901; AVX512BW-NEXT: retq 1902 %za = zext <512 x i8> %a to <512 x i16> 1903 %zb = zext <512 x i8> %b to <512 x i16> 1904 %add = add nuw nsw <512 x i16> %za, %zb 1905 %add1 = add nuw nsw <512 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> 1906 %lshr = lshr <512 x i16> %add1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> 1907 %res = trunc <512 x i16> %lshr to <512 x i8> 1908 ret <512 x i8> %res 1909} 1910 1911; This is not an avg, but its structurally similar and previously caused a crash 1912; because the constants can't be read with APInt::getZExtValue. 1913define void @not_avg_v16i8_wide_constants(<16 x i8>* %a, <16 x i8>* %b) nounwind { 1914; SSE2-LABEL: not_avg_v16i8_wide_constants: 1915; SSE2: # %bb.0: 1916; SSE2-NEXT: pushq %rbp 1917; SSE2-NEXT: pushq %r15 1918; SSE2-NEXT: pushq %r14 1919; SSE2-NEXT: pushq %r13 1920; SSE2-NEXT: pushq %r12 1921; SSE2-NEXT: pushq %rbx 1922; SSE2-NEXT: movaps (%rdi), %xmm0 1923; SSE2-NEXT: movaps (%rsi), %xmm1 1924; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp) 1925; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax 1926; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 1927; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %r13d 1928; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax 1929; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 1930; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax 1931; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 1932; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %r14d 1933; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %r15d 1934; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %r12d 1935; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %r11d 1936; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %r10d 1937; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %r9d 1938; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx 1939; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax 1940; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %edi 1941; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %ebp 1942; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %ebx 1943; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx 1944; SSE2-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp) 1945; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi 1946; SSE2-NEXT: leal -1(%rdx,%rsi), %edx 1947; SSE2-NEXT: movl %edx, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill 1948; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx 1949; SSE2-NEXT: leal -1(%rbx,%rdx), %edx 1950; SSE2-NEXT: movl %edx, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill 1951; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx 1952; SSE2-NEXT: leal -1(%rbp,%rdx), %edx 1953; SSE2-NEXT: movl %edx, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill 1954; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx 1955; SSE2-NEXT: leal -1(%rdi,%rdx), %r8d 1956; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx 1957; SSE2-NEXT: leal -1(%rax,%rdx), %edi 1958; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax 1959; SSE2-NEXT: leal -1(%rcx,%rax), %edx 1960; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax 1961; SSE2-NEXT: leal -1(%r9,%rax), %ecx 1962; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi 1963; SSE2-NEXT: leal -1(%r10,%rsi), %eax 1964; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi 1965; SSE2-NEXT: leaq -1(%r11,%rsi), %rsi 1966; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %ebx 1967; SSE2-NEXT: leaq -1(%r12,%rbx), %r12 1968; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %ebx 1969; SSE2-NEXT: leaq -1(%r15,%rbx), %r15 1970; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %ebx 1971; SSE2-NEXT: leaq -1(%r14,%rbx), %r14 1972; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %ebx 1973; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload 1974; SSE2-NEXT: leaq -1(%rbp,%rbx), %r11 1975; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %ebx 1976; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload 1977; SSE2-NEXT: leaq -1(%rbp,%rbx), %r10 1978; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %ebx 1979; SSE2-NEXT: leaq -1(%r13,%rbx), %r9 1980; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %ebx 1981; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload 1982; SSE2-NEXT: leaq -1(%r13,%rbx), %rbx 1983; SSE2-NEXT: shrl %eax 1984; SSE2-NEXT: movd %eax, %xmm8 1985; SSE2-NEXT: shrl %ecx 1986; SSE2-NEXT: movd %ecx, %xmm15 1987; SSE2-NEXT: shrl %edx 1988; SSE2-NEXT: movd %edx, %xmm9 1989; SSE2-NEXT: shrl %edi 1990; SSE2-NEXT: movd %edi, %xmm2 1991; SSE2-NEXT: shrl %r8d 1992; SSE2-NEXT: movd %r8d, %xmm10 1993; SSE2-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload 1994; SSE2-NEXT: shrl %eax 1995; SSE2-NEXT: movd %eax, %xmm6 1996; SSE2-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload 1997; SSE2-NEXT: shrl %eax 1998; SSE2-NEXT: movd %eax, %xmm11 1999; SSE2-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload 2000; SSE2-NEXT: shrl %eax 2001; SSE2-NEXT: movd %eax, %xmm4 2002; SSE2-NEXT: shrq %rsi 2003; SSE2-NEXT: movd %esi, %xmm12 2004; SSE2-NEXT: shrq %r12 2005; SSE2-NEXT: movd %r12d, %xmm3 2006; SSE2-NEXT: shrq %r15 2007; SSE2-NEXT: movd %r15d, %xmm13 2008; SSE2-NEXT: shrq %r14 2009; SSE2-NEXT: movd %r14d, %xmm7 2010; SSE2-NEXT: shrq %r11 2011; SSE2-NEXT: movd %r11d, %xmm14 2012; SSE2-NEXT: shrq %r10 2013; SSE2-NEXT: movd %r10d, %xmm5 2014; SSE2-NEXT: shrq %r9 2015; SSE2-NEXT: movd %r9d, %xmm0 2016; SSE2-NEXT: shrq %rbx 2017; SSE2-NEXT: movd %ebx, %xmm1 2018; SSE2-NEXT: punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm8[0],xmm15[1],xmm8[1],xmm15[2],xmm8[2],xmm15[3],xmm8[3],xmm15[4],xmm8[4],xmm15[5],xmm8[5],xmm15[6],xmm8[6],xmm15[7],xmm8[7] 2019; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3],xmm2[4],xmm9[4],xmm2[5],xmm9[5],xmm2[6],xmm9[6],xmm2[7],xmm9[7] 2020; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm15[0],xmm2[1],xmm15[1],xmm2[2],xmm15[2],xmm2[3],xmm15[3] 2021; SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm10[0],xmm6[1],xmm10[1],xmm6[2],xmm10[2],xmm6[3],xmm10[3],xmm6[4],xmm10[4],xmm6[5],xmm10[5],xmm6[6],xmm10[6],xmm6[7],xmm10[7] 2022; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm11[0],xmm4[1],xmm11[1],xmm4[2],xmm11[2],xmm4[3],xmm11[3],xmm4[4],xmm11[4],xmm4[5],xmm11[5],xmm4[6],xmm11[6],xmm4[7],xmm11[7] 2023; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3] 2024; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1] 2025; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm12[0],xmm3[1],xmm12[1],xmm3[2],xmm12[2],xmm3[3],xmm12[3],xmm3[4],xmm12[4],xmm3[5],xmm12[5],xmm3[6],xmm12[6],xmm3[7],xmm12[7] 2026; SSE2-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm13[0],xmm7[1],xmm13[1],xmm7[2],xmm13[2],xmm7[3],xmm13[3],xmm7[4],xmm13[4],xmm7[5],xmm13[5],xmm7[6],xmm13[6],xmm7[7],xmm13[7] 2027; SSE2-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm3[0],xmm7[1],xmm3[1],xmm7[2],xmm3[2],xmm7[3],xmm3[3] 2028; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm14[0],xmm5[1],xmm14[1],xmm5[2],xmm14[2],xmm5[3],xmm14[3],xmm5[4],xmm14[4],xmm5[5],xmm14[5],xmm5[6],xmm14[6],xmm5[7],xmm14[7] 2029; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] 2030; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3] 2031; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm7[0],xmm1[1],xmm7[1] 2032; SSE2-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm1[0] 2033; SSE2-NEXT: movdqu %xmm4, (%rax) 2034; SSE2-NEXT: popq %rbx 2035; SSE2-NEXT: popq %r12 2036; SSE2-NEXT: popq %r13 2037; SSE2-NEXT: popq %r14 2038; SSE2-NEXT: popq %r15 2039; SSE2-NEXT: popq %rbp 2040; SSE2-NEXT: retq 2041; 2042; AVX1-LABEL: not_avg_v16i8_wide_constants: 2043; AVX1: # %bb.0: 2044; AVX1-NEXT: pushq %rbp 2045; AVX1-NEXT: pushq %r15 2046; AVX1-NEXT: pushq %r14 2047; AVX1-NEXT: pushq %r13 2048; AVX1-NEXT: pushq %r12 2049; AVX1-NEXT: pushq %rbx 2050; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 2051; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 2052; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 2053; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 2054; AVX1-NEXT: vpxor %xmm5, %xmm5, %xmm5 2055; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm7 = xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7] 2056; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7] 2057; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm2[2],xmm5[2],xmm2[3],xmm5[3] 2058; AVX1-NEXT: vpextrq $1, %xmm6, %r15 2059; AVX1-NEXT: vmovq %xmm6, %r12 2060; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero 2061; AVX1-NEXT: vpextrq $1, %xmm2, %r11 2062; AVX1-NEXT: vmovq %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill 2063; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero 2064; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm2[2],xmm5[2],xmm2[3],xmm5[3] 2065; AVX1-NEXT: vpextrq $1, %xmm2, %r13 2066; AVX1-NEXT: vmovq %xmm2, %r14 2067; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero 2068; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero 2069; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm8 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero 2070; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm5[4],xmm0[5],xmm5[5],xmm0[6],xmm5[6],xmm0[7],xmm5[7] 2071; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm1[2],xmm5[2],xmm1[3],xmm5[3] 2072; AVX1-NEXT: vpextrq $1, %xmm6, %rbx 2073; AVX1-NEXT: vmovq %xmm6, %rdx 2074; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm6 = xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7] 2075; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero 2076; AVX1-NEXT: vpextrq $1, %xmm1, %r9 2077; AVX1-NEXT: vmovq %xmm1, %r10 2078; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero 2079; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm1[2],xmm5[2],xmm1[3],xmm5[3] 2080; AVX1-NEXT: vmovd %xmm7, %esi 2081; AVX1-NEXT: vpextrd $1, %xmm7, %edi 2082; AVX1-NEXT: vpextrd $2, %xmm7, %ecx 2083; AVX1-NEXT: vpextrd $3, %xmm7, %ebp 2084; AVX1-NEXT: vpextrd $3, %xmm6, %eax 2085; AVX1-NEXT: leal -1(%rbp,%rax), %eax 2086; AVX1-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill 2087; AVX1-NEXT: vpextrd $2, %xmm6, %eax 2088; AVX1-NEXT: leal -1(%rcx,%rax), %eax 2089; AVX1-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill 2090; AVX1-NEXT: vpextrd $1, %xmm6, %eax 2091; AVX1-NEXT: leal -1(%rdi,%rax), %eax 2092; AVX1-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill 2093; AVX1-NEXT: vmovd %xmm6, %eax 2094; AVX1-NEXT: leal -1(%rsi,%rax), %r8d 2095; AVX1-NEXT: vpextrq $1, %xmm5, %rax 2096; AVX1-NEXT: leal -1(%r15,%rbx), %r15d 2097; AVX1-NEXT: vmovq %xmm5, %rsi 2098; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero 2099; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero 2100; AVX1-NEXT: leal -1(%r12,%rdx), %edx 2101; AVX1-NEXT: vmovd %xmm4, %r12d 2102; AVX1-NEXT: leal -1(%r11,%r9), %r11d 2103; AVX1-NEXT: vpextrd $2, %xmm4, %edi 2104; AVX1-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload 2105; AVX1-NEXT: leal -1(%rcx,%r10), %r10d 2106; AVX1-NEXT: vpextrd $2, %xmm2, %ebx 2107; AVX1-NEXT: leal -1(%r13,%rax), %r9d 2108; AVX1-NEXT: vpextrd $3, %xmm2, %eax 2109; AVX1-NEXT: leal -1(%r14,%rsi), %esi 2110; AVX1-NEXT: vpextrd $3, %xmm1, %ecx 2111; AVX1-NEXT: leal -1(%rax,%rcx), %eax 2112; AVX1-NEXT: vpextrd $2, %xmm1, %ecx 2113; AVX1-NEXT: leal -1(%rbx,%rcx), %ebx 2114; AVX1-NEXT: vpextrd $2, %xmm3, %ecx 2115; AVX1-NEXT: leal -1(%rdi,%rcx), %ecx 2116; AVX1-NEXT: vmovd %xmm3, %edi 2117; AVX1-NEXT: leal -1(%r12,%rdi), %edi 2118; AVX1-NEXT: vpextrq $1, %xmm8, %r12 2119; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero 2120; AVX1-NEXT: vpextrq $1, %xmm0, %r13 2121; AVX1-NEXT: leal -1(%r12,%r13), %r12d 2122; AVX1-NEXT: vmovq %xmm8, %r13 2123; AVX1-NEXT: vmovq %xmm0, %r14 2124; AVX1-NEXT: leal -1(%r13,%r14), %ebp 2125; AVX1-NEXT: shrl %ebp 2126; AVX1-NEXT: vmovd %ebp, %xmm0 2127; AVX1-NEXT: shrl %r12d 2128; AVX1-NEXT: vpinsrb $1, %r12d, %xmm0, %xmm0 2129; AVX1-NEXT: shrl %esi 2130; AVX1-NEXT: vpinsrb $2, %esi, %xmm0, %xmm0 2131; AVX1-NEXT: shrl %r9d 2132; AVX1-NEXT: vpinsrb $3, %r9d, %xmm0, %xmm0 2133; AVX1-NEXT: shrl %r10d 2134; AVX1-NEXT: vpinsrb $4, %r10d, %xmm0, %xmm0 2135; AVX1-NEXT: shrl %r11d 2136; AVX1-NEXT: vpinsrb $5, %r11d, %xmm0, %xmm0 2137; AVX1-NEXT: shrl %edx 2138; AVX1-NEXT: vpinsrb $6, %edx, %xmm0, %xmm0 2139; AVX1-NEXT: shrl %r15d 2140; AVX1-NEXT: vpinsrb $7, %r15d, %xmm0, %xmm0 2141; AVX1-NEXT: shrl %edi 2142; AVX1-NEXT: vpinsrb $8, %edi, %xmm0, %xmm0 2143; AVX1-NEXT: shrl %ecx 2144; AVX1-NEXT: vpinsrb $9, %ecx, %xmm0, %xmm0 2145; AVX1-NEXT: shrl %ebx 2146; AVX1-NEXT: vpinsrb $10, %ebx, %xmm0, %xmm0 2147; AVX1-NEXT: shrl %eax 2148; AVX1-NEXT: vpinsrb $11, %eax, %xmm0, %xmm0 2149; AVX1-NEXT: shrl %r8d 2150; AVX1-NEXT: vpinsrb $12, %r8d, %xmm0, %xmm0 2151; AVX1-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload 2152; AVX1-NEXT: shrl %eax 2153; AVX1-NEXT: vpinsrb $13, %eax, %xmm0, %xmm0 2154; AVX1-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload 2155; AVX1-NEXT: shrl %eax 2156; AVX1-NEXT: vpinsrb $14, %eax, %xmm0, %xmm0 2157; AVX1-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload 2158; AVX1-NEXT: shrl %eax 2159; AVX1-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0 2160; AVX1-NEXT: vmovdqu %xmm0, (%rax) 2161; AVX1-NEXT: popq %rbx 2162; AVX1-NEXT: popq %r12 2163; AVX1-NEXT: popq %r13 2164; AVX1-NEXT: popq %r14 2165; AVX1-NEXT: popq %r15 2166; AVX1-NEXT: popq %rbp 2167; AVX1-NEXT: retq 2168; 2169; AVX2-LABEL: not_avg_v16i8_wide_constants: 2170; AVX2: # %bb.0: 2171; AVX2-NEXT: pushq %rbp 2172; AVX2-NEXT: pushq %r15 2173; AVX2-NEXT: pushq %r14 2174; AVX2-NEXT: pushq %r13 2175; AVX2-NEXT: pushq %r12 2176; AVX2-NEXT: pushq %rbx 2177; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero 2178; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero 2179; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero 2180; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm10 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero 2181; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm1 2182; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero 2183; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm3 2184; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm5 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero 2185; AVX2-NEXT: vextracti128 $1, %ymm5, %xmm4 2186; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm9 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero 2187; AVX2-NEXT: vextracti128 $1, %ymm9, %xmm7 2188; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm1 2189; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero 2190; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2 2191; AVX2-NEXT: vpextrq $1, %xmm2, %r15 2192; AVX2-NEXT: vmovq %xmm2, %r14 2193; AVX2-NEXT: vpextrq $1, %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill 2194; AVX2-NEXT: vmovq %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill 2195; AVX2-NEXT: vextracti128 $1, %ymm10, %xmm1 2196; AVX2-NEXT: vpextrq $1, %xmm1, %r13 2197; AVX2-NEXT: vmovq %xmm1, %r11 2198; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero 2199; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm11 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero 2200; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0 2201; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero 2202; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 2203; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm8 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero 2204; AVX2-NEXT: vextracti128 $1, %ymm8, %xmm1 2205; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero 2206; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm6 2207; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm0 2208; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero 2209; AVX2-NEXT: vmovd %xmm9, %r12d 2210; AVX2-NEXT: vpextrd $2, %xmm9, %r9d 2211; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm0 2212; AVX2-NEXT: vmovd %xmm7, %ecx 2213; AVX2-NEXT: vpextrd $2, %xmm7, %edi 2214; AVX2-NEXT: vmovd %xmm5, %ebx 2215; AVX2-NEXT: vpextrd $2, %xmm5, %esi 2216; AVX2-NEXT: vmovd %xmm4, %edx 2217; AVX2-NEXT: vpextrd $2, %xmm4, %ebp 2218; AVX2-NEXT: vpextrd $2, %xmm1, %eax 2219; AVX2-NEXT: leal -1(%rbp,%rax), %eax 2220; AVX2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill 2221; AVX2-NEXT: vmovd %xmm1, %eax 2222; AVX2-NEXT: leal -1(%rdx,%rax), %eax 2223; AVX2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill 2224; AVX2-NEXT: vpextrd $2, %xmm8, %eax 2225; AVX2-NEXT: leal -1(%rsi,%rax), %eax 2226; AVX2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill 2227; AVX2-NEXT: vmovd %xmm8, %eax 2228; AVX2-NEXT: leal -1(%rbx,%rax), %r10d 2229; AVX2-NEXT: vpextrd $2, %xmm6, %eax 2230; AVX2-NEXT: leal -1(%rdi,%rax), %r8d 2231; AVX2-NEXT: vmovd %xmm6, %eax 2232; AVX2-NEXT: leal -1(%rcx,%rax), %edi 2233; AVX2-NEXT: vpextrd $2, %xmm3, %eax 2234; AVX2-NEXT: leal -1(%r9,%rax), %r9d 2235; AVX2-NEXT: vmovd %xmm3, %ecx 2236; AVX2-NEXT: leal -1(%r12,%rcx), %r12d 2237; AVX2-NEXT: vpextrq $1, %xmm0, %rcx 2238; AVX2-NEXT: leal -1(%r15,%rcx), %r15d 2239; AVX2-NEXT: vmovq %xmm0, %rcx 2240; AVX2-NEXT: leal -1(%r14,%rcx), %r14d 2241; AVX2-NEXT: vpextrq $1, %xmm2, %rdx 2242; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload 2243; AVX2-NEXT: leal -1(%rax,%rdx), %edx 2244; AVX2-NEXT: vmovq %xmm2, %rax 2245; AVX2-NEXT: vextracti128 $1, %ymm11, %xmm0 2246; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload 2247; AVX2-NEXT: leal -1(%rcx,%rax), %eax 2248; AVX2-NEXT: vpextrq $1, %xmm0, %rsi 2249; AVX2-NEXT: leal -1(%r13,%rsi), %esi 2250; AVX2-NEXT: vmovq %xmm0, %rbx 2251; AVX2-NEXT: leal -1(%r11,%rbx), %ebx 2252; AVX2-NEXT: vpextrq $1, %xmm10, %rcx 2253; AVX2-NEXT: vpextrq $1, %xmm11, %r13 2254; AVX2-NEXT: leal -1(%rcx,%r13), %ecx 2255; AVX2-NEXT: vmovq %xmm10, %r13 2256; AVX2-NEXT: vmovq %xmm11, %r11 2257; AVX2-NEXT: leaq -1(%r13,%r11), %rbp 2258; AVX2-NEXT: shrq %rbp 2259; AVX2-NEXT: vmovd %ebp, %xmm0 2260; AVX2-NEXT: shrl %ecx 2261; AVX2-NEXT: vpinsrb $1, %ecx, %xmm0, %xmm0 2262; AVX2-NEXT: shrl %ebx 2263; AVX2-NEXT: vpinsrb $2, %ebx, %xmm0, %xmm0 2264; AVX2-NEXT: shrl %esi 2265; AVX2-NEXT: vpinsrb $3, %esi, %xmm0, %xmm0 2266; AVX2-NEXT: shrl %eax 2267; AVX2-NEXT: vpinsrb $4, %eax, %xmm0, %xmm0 2268; AVX2-NEXT: shrl %edx 2269; AVX2-NEXT: vpinsrb $5, %edx, %xmm0, %xmm0 2270; AVX2-NEXT: shrl %r14d 2271; AVX2-NEXT: vpinsrb $6, %r14d, %xmm0, %xmm0 2272; AVX2-NEXT: shrl %r15d 2273; AVX2-NEXT: vpinsrb $7, %r15d, %xmm0, %xmm0 2274; AVX2-NEXT: shrl %r12d 2275; AVX2-NEXT: vpinsrb $8, %r12d, %xmm0, %xmm0 2276; AVX2-NEXT: shrl %r9d 2277; AVX2-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0 2278; AVX2-NEXT: shrl %edi 2279; AVX2-NEXT: vpinsrb $10, %edi, %xmm0, %xmm0 2280; AVX2-NEXT: shrl %r8d 2281; AVX2-NEXT: vpinsrb $11, %r8d, %xmm0, %xmm0 2282; AVX2-NEXT: shrl %r10d 2283; AVX2-NEXT: vpinsrb $12, %r10d, %xmm0, %xmm0 2284; AVX2-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload 2285; AVX2-NEXT: shrl %eax 2286; AVX2-NEXT: vpinsrb $13, %eax, %xmm0, %xmm0 2287; AVX2-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload 2288; AVX2-NEXT: shrl %eax 2289; AVX2-NEXT: vpinsrb $14, %eax, %xmm0, %xmm0 2290; AVX2-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload 2291; AVX2-NEXT: shrl %eax 2292; AVX2-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0 2293; AVX2-NEXT: vmovdqu %xmm0, (%rax) 2294; AVX2-NEXT: popq %rbx 2295; AVX2-NEXT: popq %r12 2296; AVX2-NEXT: popq %r13 2297; AVX2-NEXT: popq %r14 2298; AVX2-NEXT: popq %r15 2299; AVX2-NEXT: popq %rbp 2300; AVX2-NEXT: vzeroupper 2301; AVX2-NEXT: retq 2302; 2303; AVX512-LABEL: not_avg_v16i8_wide_constants: 2304; AVX512: # %bb.0: 2305; AVX512-NEXT: pushq %rbp 2306; AVX512-NEXT: pushq %r15 2307; AVX512-NEXT: pushq %r14 2308; AVX512-NEXT: pushq %r13 2309; AVX512-NEXT: pushq %r12 2310; AVX512-NEXT: pushq %rbx 2311; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero 2312; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero 2313; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero 2314; AVX512-NEXT: vpmovzxdq {{.*#+}} ymm10 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero 2315; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm1 2316; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero 2317; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm3 2318; AVX512-NEXT: vpmovzxdq {{.*#+}} ymm5 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero 2319; AVX512-NEXT: vextracti128 $1, %ymm5, %xmm4 2320; AVX512-NEXT: vpmovzxdq {{.*#+}} ymm9 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero 2321; AVX512-NEXT: vextracti128 $1, %ymm9, %xmm7 2322; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm1 2323; AVX512-NEXT: vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero 2324; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm2 2325; AVX512-NEXT: vpextrq $1, %xmm2, %r15 2326; AVX512-NEXT: vmovq %xmm2, %r14 2327; AVX512-NEXT: vpextrq $1, %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill 2328; AVX512-NEXT: vmovq %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill 2329; AVX512-NEXT: vextracti128 $1, %ymm10, %xmm1 2330; AVX512-NEXT: vpextrq $1, %xmm1, %r13 2331; AVX512-NEXT: vmovq %xmm1, %r11 2332; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero 2333; AVX512-NEXT: vpmovzxdq {{.*#+}} ymm11 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero 2334; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm0 2335; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero 2336; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1 2337; AVX512-NEXT: vpmovzxdq {{.*#+}} ymm8 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero 2338; AVX512-NEXT: vextracti128 $1, %ymm8, %xmm1 2339; AVX512-NEXT: vpmovzxdq {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero 2340; AVX512-NEXT: vextracti128 $1, %ymm3, %xmm6 2341; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm0 2342; AVX512-NEXT: vpmovzxdq {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero 2343; AVX512-NEXT: vmovd %xmm9, %r12d 2344; AVX512-NEXT: vpextrd $2, %xmm9, %r9d 2345; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm0 2346; AVX512-NEXT: vmovd %xmm7, %ecx 2347; AVX512-NEXT: vpextrd $2, %xmm7, %edi 2348; AVX512-NEXT: vmovd %xmm5, %ebx 2349; AVX512-NEXT: vpextrd $2, %xmm5, %esi 2350; AVX512-NEXT: vmovd %xmm4, %edx 2351; AVX512-NEXT: vpextrd $2, %xmm4, %ebp 2352; AVX512-NEXT: vpextrd $2, %xmm1, %eax 2353; AVX512-NEXT: leal -1(%rbp,%rax), %eax 2354; AVX512-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill 2355; AVX512-NEXT: vmovd %xmm1, %eax 2356; AVX512-NEXT: leal -1(%rdx,%rax), %eax 2357; AVX512-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill 2358; AVX512-NEXT: vpextrd $2, %xmm8, %eax 2359; AVX512-NEXT: leal -1(%rsi,%rax), %eax 2360; AVX512-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill 2361; AVX512-NEXT: vmovd %xmm8, %eax 2362; AVX512-NEXT: leal -1(%rbx,%rax), %r10d 2363; AVX512-NEXT: vpextrd $2, %xmm6, %eax 2364; AVX512-NEXT: leal -1(%rdi,%rax), %r8d 2365; AVX512-NEXT: vmovd %xmm6, %eax 2366; AVX512-NEXT: leal -1(%rcx,%rax), %edi 2367; AVX512-NEXT: vpextrd $2, %xmm3, %eax 2368; AVX512-NEXT: leal -1(%r9,%rax), %r9d 2369; AVX512-NEXT: vmovd %xmm3, %ecx 2370; AVX512-NEXT: leal -1(%r12,%rcx), %r12d 2371; AVX512-NEXT: vpextrq $1, %xmm0, %rcx 2372; AVX512-NEXT: leal -1(%r15,%rcx), %r15d 2373; AVX512-NEXT: vmovq %xmm0, %rcx 2374; AVX512-NEXT: leal -1(%r14,%rcx), %r14d 2375; AVX512-NEXT: vpextrq $1, %xmm2, %rdx 2376; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload 2377; AVX512-NEXT: leal -1(%rax,%rdx), %edx 2378; AVX512-NEXT: vmovq %xmm2, %rax 2379; AVX512-NEXT: vextracti128 $1, %ymm11, %xmm0 2380; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload 2381; AVX512-NEXT: leal -1(%rcx,%rax), %eax 2382; AVX512-NEXT: vpextrq $1, %xmm0, %rsi 2383; AVX512-NEXT: leal -1(%r13,%rsi), %esi 2384; AVX512-NEXT: vmovq %xmm0, %rbx 2385; AVX512-NEXT: leal -1(%r11,%rbx), %ebx 2386; AVX512-NEXT: vpextrq $1, %xmm10, %rcx 2387; AVX512-NEXT: vpextrq $1, %xmm11, %r13 2388; AVX512-NEXT: leal -1(%rcx,%r13), %ecx 2389; AVX512-NEXT: vmovq %xmm10, %r13 2390; AVX512-NEXT: vmovq %xmm11, %r11 2391; AVX512-NEXT: leaq -1(%r13,%r11), %rbp 2392; AVX512-NEXT: shrq %rbp 2393; AVX512-NEXT: vmovd %ebp, %xmm0 2394; AVX512-NEXT: shrl %ecx 2395; AVX512-NEXT: vpinsrb $1, %ecx, %xmm0, %xmm0 2396; AVX512-NEXT: shrl %ebx 2397; AVX512-NEXT: vpinsrb $2, %ebx, %xmm0, %xmm0 2398; AVX512-NEXT: shrl %esi 2399; AVX512-NEXT: vpinsrb $3, %esi, %xmm0, %xmm0 2400; AVX512-NEXT: shrl %eax 2401; AVX512-NEXT: vpinsrb $4, %eax, %xmm0, %xmm0 2402; AVX512-NEXT: shrl %edx 2403; AVX512-NEXT: vpinsrb $5, %edx, %xmm0, %xmm0 2404; AVX512-NEXT: shrl %r14d 2405; AVX512-NEXT: vpinsrb $6, %r14d, %xmm0, %xmm0 2406; AVX512-NEXT: shrl %r15d 2407; AVX512-NEXT: vpinsrb $7, %r15d, %xmm0, %xmm0 2408; AVX512-NEXT: shrl %r12d 2409; AVX512-NEXT: vpinsrb $8, %r12d, %xmm0, %xmm0 2410; AVX512-NEXT: shrl %r9d 2411; AVX512-NEXT: vpinsrb $9, %r9d, %xmm0, %xmm0 2412; AVX512-NEXT: shrl %edi 2413; AVX512-NEXT: vpinsrb $10, %edi, %xmm0, %xmm0 2414; AVX512-NEXT: shrl %r8d 2415; AVX512-NEXT: vpinsrb $11, %r8d, %xmm0, %xmm0 2416; AVX512-NEXT: shrl %r10d 2417; AVX512-NEXT: vpinsrb $12, %r10d, %xmm0, %xmm0 2418; AVX512-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload 2419; AVX512-NEXT: shrl %eax 2420; AVX512-NEXT: vpinsrb $13, %eax, %xmm0, %xmm0 2421; AVX512-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload 2422; AVX512-NEXT: shrl %eax 2423; AVX512-NEXT: vpinsrb $14, %eax, %xmm0, %xmm0 2424; AVX512-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload 2425; AVX512-NEXT: shrl %eax 2426; AVX512-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0 2427; AVX512-NEXT: vmovdqu %xmm0, (%rax) 2428; AVX512-NEXT: popq %rbx 2429; AVX512-NEXT: popq %r12 2430; AVX512-NEXT: popq %r13 2431; AVX512-NEXT: popq %r14 2432; AVX512-NEXT: popq %r15 2433; AVX512-NEXT: popq %rbp 2434; AVX512-NEXT: vzeroupper 2435; AVX512-NEXT: retq 2436 %1 = load <16 x i8>, <16 x i8>* %a 2437 %2 = load <16 x i8>, <16 x i8>* %b 2438 %3 = zext <16 x i8> %1 to <16 x i128> 2439 %4 = zext <16 x i8> %2 to <16 x i128> 2440 %5 = add nuw nsw <16 x i128> %3, <i128 -1, i128 -1, i128 -1, i128 -1, i128 -1, i128 -1, i128 -1, i128 -1, i128 -1, i128 -1, i128 -1, i128 -1, i128 -1, i128 -1, i128 -1, i128 -1> 2441 %6 = add nuw nsw <16 x i128> %5, %4 2442 %7 = lshr <16 x i128> %6, <i128 1, i128 1, i128 1, i128 1, i128 1, i128 1, i128 1, i128 1, i128 1, i128 1, i128 1, i128 1, i128 1, i128 1, i128 1, i128 1> 2443 %8 = trunc <16 x i128> %7 to <16 x i8> 2444 store <16 x i8> %8, <16 x i8>* undef, align 4 2445 ret void 2446} 2447 2448; Make sure we don't fail on single element vectors. 2449define <1 x i8> @avg_v1i8(<1 x i8> %x, <1 x i8> %y) { 2450; SSE2-LABEL: avg_v1i8: 2451; SSE2: # %bb.0: 2452; SSE2-NEXT: movzbl %dil, %eax 2453; SSE2-NEXT: movzbl %sil, %ecx 2454; SSE2-NEXT: leal 1(%rax,%rcx), %eax 2455; SSE2-NEXT: shrl %eax 2456; SSE2-NEXT: # kill: def $al killed $al killed $eax 2457; SSE2-NEXT: retq 2458; 2459; AVX-LABEL: avg_v1i8: 2460; AVX: # %bb.0: 2461; AVX-NEXT: movzbl %dil, %eax 2462; AVX-NEXT: movzbl %sil, %ecx 2463; AVX-NEXT: leal 1(%rax,%rcx), %eax 2464; AVX-NEXT: shrl %eax 2465; AVX-NEXT: # kill: def $al killed $al killed $eax 2466; AVX-NEXT: retq 2467 %a = zext <1 x i8> %x to <1 x i16> 2468 %b = zext <1 x i8> %y to <1 x i16> 2469 %c = add <1 x i16> %a, %b 2470 %d = add <1 x i16> %c, <i16 1> 2471 %e = lshr <1 x i16> %d, <i16 1> 2472 %f = trunc <1 x i16> %e to <1 x i8> 2473 ret <1 x i8> %f 2474} 2475 2476