1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py 2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse2 | FileCheck %s --check-prefix=SSE2 3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=AVX2 4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx512f | FileCheck %s --check-prefix=AVX512F 5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx512bw | FileCheck %s --check-prefix=AVX512BW 6 7define void @avg_v4i8(<4 x i8>* %a, <4 x i8>* %b) { 8; SSE2-LABEL: avg_v4i8: 9; SSE2: # BB#0: 10; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero 11; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero 12; SSE2-NEXT: pavgb %xmm0, %xmm1 13; SSE2-NEXT: movd %xmm1, (%rax) 14; SSE2-NEXT: retq 15; 16; AVX2-LABEL: avg_v4i8: 17; AVX2: # BB#0: 18; AVX2-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero 19; AVX2-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero 20; AVX2-NEXT: vpavgb %xmm0, %xmm1, %xmm0 21; AVX2-NEXT: vmovd %xmm0, (%rax) 22; AVX2-NEXT: retq 23; 24; AVX512F-LABEL: avg_v4i8: 25; AVX512F: # BB#0: 26; AVX512F-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero 27; AVX512F-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero 28; AVX512F-NEXT: vpavgb %xmm0, %xmm1, %xmm0 29; AVX512F-NEXT: vmovd %xmm0, (%rax) 30; AVX512F-NEXT: retq 31; 32; AVX512BW-LABEL: avg_v4i8: 33; AVX512BW: # BB#0: 34; AVX512BW-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero 35; AVX512BW-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero 36; AVX512BW-NEXT: vpavgb %xmm0, %xmm1, %xmm0 37; AVX512BW-NEXT: vmovd %xmm0, (%rax) 38; AVX512BW-NEXT: retq 39 %1 = load <4 x i8>, <4 x i8>* %a 40 %2 = load <4 x i8>, <4 x i8>* %b 41 %3 = zext <4 x i8> %1 to <4 x i32> 42 %4 = zext <4 x i8> %2 to <4 x i32> 43 %5 = add nuw nsw <4 x i32> %3, <i32 1, i32 1, i32 1, i32 1> 44 %6 = add nuw nsw <4 x i32> %5, %4 45 %7 = lshr <4 x i32> %6, <i32 1, i32 1, i32 1, i32 1> 46 %8 = trunc <4 x i32> %7 to <4 x i8> 47 store <4 x i8> %8, <4 x i8>* undef, align 4 48 ret void 49} 50 51define void @avg_v8i8(<8 x i8>* %a, <8 x i8>* %b) { 52; SSE2-LABEL: avg_v8i8: 53; SSE2: # BB#0: 54; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero 55; SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero 56; SSE2-NEXT: pavgb %xmm0, %xmm1 57; SSE2-NEXT: movq %xmm1, (%rax) 58; SSE2-NEXT: retq 59; 60; AVX2-LABEL: avg_v8i8: 61; AVX2: # BB#0: 62; AVX2-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 63; AVX2-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero 64; AVX2-NEXT: vpavgb %xmm0, %xmm1, %xmm0 65; AVX2-NEXT: vmovq %xmm0, (%rax) 66; AVX2-NEXT: retq 67; 68; AVX512F-LABEL: avg_v8i8: 69; AVX512F: # BB#0: 70; AVX512F-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 71; AVX512F-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero 72; AVX512F-NEXT: vpavgb %xmm0, %xmm1, %xmm0 73; AVX512F-NEXT: vmovq %xmm0, (%rax) 74; AVX512F-NEXT: retq 75; 76; AVX512BW-LABEL: avg_v8i8: 77; AVX512BW: # BB#0: 78; AVX512BW-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 79; AVX512BW-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero 80; AVX512BW-NEXT: vpavgb %xmm0, %xmm1, %xmm0 81; AVX512BW-NEXT: vmovq %xmm0, (%rax) 82; AVX512BW-NEXT: retq 83 %1 = load <8 x i8>, <8 x i8>* %a 84 %2 = load <8 x i8>, <8 x i8>* %b 85 %3 = zext <8 x i8> %1 to <8 x i32> 86 %4 = zext <8 x i8> %2 to <8 x i32> 87 %5 = add nuw nsw <8 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 88 %6 = add nuw nsw <8 x i32> %5, %4 89 %7 = lshr <8 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 90 %8 = trunc <8 x i32> %7 to <8 x i8> 91 store <8 x i8> %8, <8 x i8>* undef, align 4 92 ret void 93} 94 95define void @avg_v16i8(<16 x i8>* %a, <16 x i8>* %b) { 96; SSE2-LABEL: avg_v16i8: 97; SSE2: # BB#0: 98; SSE2-NEXT: movdqa (%rsi), %xmm0 99; SSE2-NEXT: pavgb (%rdi), %xmm0 100; SSE2-NEXT: movdqu %xmm0, (%rax) 101; SSE2-NEXT: retq 102; 103; AVX2-LABEL: avg_v16i8: 104; AVX2: # BB#0: 105; AVX2-NEXT: vmovdqa (%rsi), %xmm0 106; AVX2-NEXT: vpavgb (%rdi), %xmm0, %xmm0 107; AVX2-NEXT: vmovdqu %xmm0, (%rax) 108; AVX2-NEXT: retq 109; 110; AVX512F-LABEL: avg_v16i8: 111; AVX512F: # BB#0: 112; AVX512F-NEXT: vmovdqa (%rsi), %xmm0 113; AVX512F-NEXT: vpavgb (%rdi), %xmm0, %xmm0 114; AVX512F-NEXT: vmovdqu %xmm0, (%rax) 115; AVX512F-NEXT: retq 116; 117; AVX512BW-LABEL: avg_v16i8: 118; AVX512BW: # BB#0: 119; AVX512BW-NEXT: vmovdqa (%rsi), %xmm0 120; AVX512BW-NEXT: vpavgb (%rdi), %xmm0, %xmm0 121; AVX512BW-NEXT: vmovdqu %xmm0, (%rax) 122; AVX512BW-NEXT: retq 123 %1 = load <16 x i8>, <16 x i8>* %a 124 %2 = load <16 x i8>, <16 x i8>* %b 125 %3 = zext <16 x i8> %1 to <16 x i32> 126 %4 = zext <16 x i8> %2 to <16 x i32> 127 %5 = add nuw nsw <16 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 128 %6 = add nuw nsw <16 x i32> %5, %4 129 %7 = lshr <16 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 130 %8 = trunc <16 x i32> %7 to <16 x i8> 131 store <16 x i8> %8, <16 x i8>* undef, align 4 132 ret void 133} 134 135define void @avg_v32i8(<32 x i8>* %a, <32 x i8>* %b) { 136; SSE2-LABEL: avg_v32i8: 137; SSE2: # BB#0: 138; SSE2-NEXT: movdqa (%rdi), %xmm8 139; SSE2-NEXT: movdqa 16(%rdi), %xmm11 140; SSE2-NEXT: movdqa (%rsi), %xmm0 141; SSE2-NEXT: movdqa 16(%rsi), %xmm1 142; SSE2-NEXT: pxor %xmm4, %xmm4 143; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm8[2,3,0,1] 144; SSE2-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1],xmm8[2],xmm4[2],xmm8[3],xmm4[3],xmm8[4],xmm4[4],xmm8[5],xmm4[5],xmm8[6],xmm4[6],xmm8[7],xmm4[7] 145; SSE2-NEXT: movdqa %xmm8, %xmm2 146; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7] 147; SSE2-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp) # 16-byte Spill 148; SSE2-NEXT: punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1],xmm8[2],xmm4[2],xmm8[3],xmm4[3] 149; SSE2-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3],xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7] 150; SSE2-NEXT: movdqa %xmm10, %xmm12 151; SSE2-NEXT: punpckhwd {{.*#+}} xmm12 = xmm12[4],xmm4[4],xmm12[5],xmm4[5],xmm12[6],xmm4[6],xmm12[7],xmm4[7] 152; SSE2-NEXT: punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3] 153; SSE2-NEXT: pshufd {{.*#+}} xmm15 = xmm11[2,3,0,1] 154; SSE2-NEXT: punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1],xmm11[2],xmm4[2],xmm11[3],xmm4[3],xmm11[4],xmm4[4],xmm11[5],xmm4[5],xmm11[6],xmm4[6],xmm11[7],xmm4[7] 155; SSE2-NEXT: movdqa %xmm11, %xmm14 156; SSE2-NEXT: punpckhwd {{.*#+}} xmm14 = xmm14[4],xmm4[4],xmm14[5],xmm4[5],xmm14[6],xmm4[6],xmm14[7],xmm4[7] 157; SSE2-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1],xmm11[2],xmm4[2],xmm11[3],xmm4[3] 158; SSE2-NEXT: punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm4[0],xmm15[1],xmm4[1],xmm15[2],xmm4[2],xmm15[3],xmm4[3],xmm15[4],xmm4[4],xmm15[5],xmm4[5],xmm15[6],xmm4[6],xmm15[7],xmm4[7] 159; SSE2-NEXT: movdqa %xmm15, %xmm9 160; SSE2-NEXT: punpckhwd {{.*#+}} xmm9 = xmm9[4],xmm4[4],xmm9[5],xmm4[5],xmm9[6],xmm4[6],xmm9[7],xmm4[7] 161; SSE2-NEXT: punpcklwd {{.*#+}} xmm15 = xmm15[0],xmm4[0],xmm15[1],xmm4[1],xmm15[2],xmm4[2],xmm15[3],xmm4[3] 162; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1] 163; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7] 164; SSE2-NEXT: movdqa %xmm0, %xmm7 165; SSE2-NEXT: punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm4[4],xmm7[5],xmm4[5],xmm7[6],xmm4[6],xmm7[7],xmm4[7] 166; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3] 167; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7] 168; SSE2-NEXT: movdqa %xmm3, %xmm6 169; SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7] 170; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3] 171; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1] 172; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] 173; SSE2-NEXT: movdqa %xmm1, %xmm5 174; SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7] 175; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3] 176; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7] 177; SSE2-NEXT: movdqa %xmm2, %xmm13 178; SSE2-NEXT: punpckhwd {{.*#+}} xmm13 = xmm13[4],xmm4[4],xmm13[5],xmm4[5],xmm13[6],xmm4[6],xmm13[7],xmm4[7] 179; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3] 180; SSE2-NEXT: paddd %xmm15, %xmm2 181; SSE2-NEXT: paddd %xmm9, %xmm13 182; SSE2-NEXT: paddd %xmm11, %xmm1 183; SSE2-NEXT: paddd %xmm14, %xmm5 184; SSE2-NEXT: paddd %xmm10, %xmm3 185; SSE2-NEXT: paddd %xmm12, %xmm6 186; SSE2-NEXT: paddd %xmm8, %xmm0 187; SSE2-NEXT: paddd -{{[0-9]+}}(%rsp), %xmm7 # 16-byte Folded Reload 188; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [1,1,1,1] 189; SSE2-NEXT: paddd %xmm4, %xmm7 190; SSE2-NEXT: paddd %xmm4, %xmm0 191; SSE2-NEXT: paddd %xmm4, %xmm6 192; SSE2-NEXT: paddd %xmm4, %xmm3 193; SSE2-NEXT: paddd %xmm4, %xmm5 194; SSE2-NEXT: paddd %xmm4, %xmm1 195; SSE2-NEXT: paddd %xmm4, %xmm13 196; SSE2-NEXT: paddd %xmm4, %xmm2 197; SSE2-NEXT: psrld $1, %xmm0 198; SSE2-NEXT: psrld $1, %xmm7 199; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0] 200; SSE2-NEXT: pand %xmm4, %xmm7 201; SSE2-NEXT: pand %xmm4, %xmm0 202; SSE2-NEXT: packuswb %xmm7, %xmm0 203; SSE2-NEXT: psrld $1, %xmm3 204; SSE2-NEXT: psrld $1, %xmm6 205; SSE2-NEXT: pand %xmm4, %xmm6 206; SSE2-NEXT: pand %xmm4, %xmm3 207; SSE2-NEXT: packuswb %xmm6, %xmm3 208; SSE2-NEXT: packuswb %xmm3, %xmm0 209; SSE2-NEXT: psrld $1, %xmm1 210; SSE2-NEXT: psrld $1, %xmm5 211; SSE2-NEXT: pand %xmm4, %xmm5 212; SSE2-NEXT: pand %xmm4, %xmm1 213; SSE2-NEXT: packuswb %xmm5, %xmm1 214; SSE2-NEXT: psrld $1, %xmm2 215; SSE2-NEXT: psrld $1, %xmm13 216; SSE2-NEXT: pand %xmm4, %xmm13 217; SSE2-NEXT: pand %xmm4, %xmm2 218; SSE2-NEXT: packuswb %xmm13, %xmm2 219; SSE2-NEXT: packuswb %xmm2, %xmm1 220; SSE2-NEXT: movdqu %xmm1, (%rax) 221; SSE2-NEXT: movdqu %xmm0, (%rax) 222; SSE2-NEXT: retq 223; 224; AVX2-LABEL: avg_v32i8: 225; AVX2: # BB#0: 226; AVX2-NEXT: vmovdqa (%rsi), %ymm0 227; AVX2-NEXT: vpavgb (%rdi), %ymm0, %ymm0 228; AVX2-NEXT: vmovdqu %ymm0, (%rax) 229; AVX2-NEXT: vzeroupper 230; AVX2-NEXT: retq 231; 232; AVX512F-LABEL: avg_v32i8: 233; AVX512F: # BB#0: 234; AVX512F-NEXT: vmovdqa (%rsi), %ymm0 235; AVX512F-NEXT: vpavgb (%rdi), %ymm0, %ymm0 236; AVX512F-NEXT: vmovdqu %ymm0, (%rax) 237; AVX512F-NEXT: retq 238; 239; AVX512BW-LABEL: avg_v32i8: 240; AVX512BW: # BB#0: 241; AVX512BW-NEXT: vmovdqa (%rsi), %ymm0 242; AVX512BW-NEXT: vpavgb (%rdi), %ymm0, %ymm0 243; AVX512BW-NEXT: vmovdqu %ymm0, (%rax) 244; AVX512BW-NEXT: retq 245 %1 = load <32 x i8>, <32 x i8>* %a 246 %2 = load <32 x i8>, <32 x i8>* %b 247 %3 = zext <32 x i8> %1 to <32 x i32> 248 %4 = zext <32 x i8> %2 to <32 x i32> 249 %5 = add nuw nsw <32 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 250 %6 = add nuw nsw <32 x i32> %5, %4 251 %7 = lshr <32 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 252 %8 = trunc <32 x i32> %7 to <32 x i8> 253 store <32 x i8> %8, <32 x i8>* undef, align 4 254 ret void 255} 256 257define void @avg_v64i8(<64 x i8>* %a, <64 x i8>* %b) { 258; SSE2-LABEL: avg_v64i8: 259; SSE2: # BB#0: 260; SSE2-NEXT: subq $152, %rsp 261; SSE2-NEXT: .Lcfi0: 262; SSE2-NEXT: .cfi_def_cfa_offset 160 263; SSE2-NEXT: movdqa (%rdi), %xmm1 264; SSE2-NEXT: movdqa 16(%rdi), %xmm2 265; SSE2-NEXT: movdqa 32(%rdi), %xmm5 266; SSE2-NEXT: movdqa 48(%rdi), %xmm6 267; SSE2-NEXT: pxor %xmm0, %xmm0 268; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[2,3,0,1] 269; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] 270; SSE2-NEXT: movdqa %xmm1, %xmm3 271; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] 272; SSE2-NEXT: movdqa %xmm3, {{[0-9]+}}(%rsp) # 16-byte Spill 273; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] 274; SSE2-NEXT: movdqa %xmm1, {{[0-9]+}}(%rsp) # 16-byte Spill 275; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7] 276; SSE2-NEXT: movdqa %xmm4, %xmm3 277; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] 278; SSE2-NEXT: movdqa %xmm3, {{[0-9]+}}(%rsp) # 16-byte Spill 279; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3] 280; SSE2-NEXT: movdqa %xmm4, {{[0-9]+}}(%rsp) # 16-byte Spill 281; SSE2-NEXT: movdqa %xmm2, %xmm1 282; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1] 283; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] 284; SSE2-NEXT: movdqa %xmm1, %xmm2 285; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] 286; SSE2-NEXT: movdqa %xmm2, {{[0-9]+}}(%rsp) # 16-byte Spill 287; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] 288; SSE2-NEXT: movdqa %xmm1, {{[0-9]+}}(%rsp) # 16-byte Spill 289; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] 290; SSE2-NEXT: movdqa %xmm3, %xmm2 291; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] 292; SSE2-NEXT: movdqa %xmm2, {{[0-9]+}}(%rsp) # 16-byte Spill 293; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3] 294; SSE2-NEXT: movdqa %xmm3, {{[0-9]+}}(%rsp) # 16-byte Spill 295; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm5[2,3,0,1] 296; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7] 297; SSE2-NEXT: movdqa %xmm5, %xmm2 298; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] 299; SSE2-NEXT: movdqa %xmm2, (%rsp) # 16-byte Spill 300; SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3] 301; SSE2-NEXT: movdqa %xmm5, -{{[0-9]+}}(%rsp) # 16-byte Spill 302; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] 303; SSE2-NEXT: movdqa %xmm3, %xmm2 304; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] 305; SSE2-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp) # 16-byte Spill 306; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3] 307; SSE2-NEXT: movdqa %xmm3, -{{[0-9]+}}(%rsp) # 16-byte Spill 308; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm6[2,3,0,1] 309; SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3],xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7] 310; SSE2-NEXT: movdqa %xmm6, %xmm1 311; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] 312; SSE2-NEXT: movdqa %xmm1, -{{[0-9]+}}(%rsp) # 16-byte Spill 313; SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3] 314; SSE2-NEXT: movdqa %xmm6, -{{[0-9]+}}(%rsp) # 16-byte Spill 315; SSE2-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm0[0],xmm9[1],xmm0[1],xmm9[2],xmm0[2],xmm9[3],xmm0[3],xmm9[4],xmm0[4],xmm9[5],xmm0[5],xmm9[6],xmm0[6],xmm9[7],xmm0[7] 316; SSE2-NEXT: movdqa %xmm9, %xmm1 317; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] 318; SSE2-NEXT: movdqa %xmm1, -{{[0-9]+}}(%rsp) # 16-byte Spill 319; SSE2-NEXT: punpcklwd {{.*#+}} xmm9 = xmm9[0],xmm0[0],xmm9[1],xmm0[1],xmm9[2],xmm0[2],xmm9[3],xmm0[3] 320; SSE2-NEXT: movdqa (%rsi), %xmm15 321; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm15[2,3,0,1] 322; SSE2-NEXT: punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3],xmm15[4],xmm0[4],xmm15[5],xmm0[5],xmm15[6],xmm0[6],xmm15[7],xmm0[7] 323; SSE2-NEXT: movdqa %xmm15, %xmm10 324; SSE2-NEXT: punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm0[4],xmm10[5],xmm0[5],xmm10[6],xmm0[6],xmm10[7],xmm0[7] 325; SSE2-NEXT: punpcklwd {{.*#+}} xmm15 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3] 326; SSE2-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3],xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7] 327; SSE2-NEXT: movdqa %xmm7, %xmm14 328; SSE2-NEXT: punpckhwd {{.*#+}} xmm14 = xmm14[4],xmm0[4],xmm14[5],xmm0[5],xmm14[6],xmm0[6],xmm14[7],xmm0[7] 329; SSE2-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3] 330; SSE2-NEXT: movdqa 16(%rsi), %xmm1 331; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm1[2,3,0,1] 332; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] 333; SSE2-NEXT: movdqa %xmm1, %xmm13 334; SSE2-NEXT: punpckhwd {{.*#+}} xmm13 = xmm13[4],xmm0[4],xmm13[5],xmm0[5],xmm13[6],xmm0[6],xmm13[7],xmm0[7] 335; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] 336; SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3],xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7] 337; SSE2-NEXT: movdqa %xmm6, %xmm12 338; SSE2-NEXT: punpckhwd {{.*#+}} xmm12 = xmm12[4],xmm0[4],xmm12[5],xmm0[5],xmm12[6],xmm0[6],xmm12[7],xmm0[7] 339; SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3] 340; SSE2-NEXT: movdqa 32(%rsi), %xmm2 341; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm2[2,3,0,1] 342; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] 343; SSE2-NEXT: movdqa %xmm2, %xmm11 344; SSE2-NEXT: punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm0[4],xmm11[5],xmm0[5],xmm11[6],xmm0[6],xmm11[7],xmm0[7] 345; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3] 346; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7] 347; SSE2-NEXT: movdqa %xmm5, %xmm3 348; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] 349; SSE2-NEXT: movdqa %xmm3, -{{[0-9]+}}(%rsp) # 16-byte Spill 350; SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3] 351; SSE2-NEXT: movdqa 48(%rsi), %xmm3 352; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm3[2,3,0,1] 353; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] 354; SSE2-NEXT: movdqa %xmm3, %xmm4 355; SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7] 356; SSE2-NEXT: movdqa %xmm4, -{{[0-9]+}}(%rsp) # 16-byte Spill 357; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3] 358; SSE2-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3],xmm8[4],xmm0[4],xmm8[5],xmm0[5],xmm8[6],xmm0[6],xmm8[7],xmm0[7] 359; SSE2-NEXT: movdqa %xmm8, %xmm4 360; SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7] 361; SSE2-NEXT: punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3] 362; SSE2-NEXT: paddd %xmm9, %xmm8 363; SSE2-NEXT: paddd -{{[0-9]+}}(%rsp), %xmm4 # 16-byte Folded Reload 364; SSE2-NEXT: movdqa %xmm4, -{{[0-9]+}}(%rsp) # 16-byte Spill 365; SSE2-NEXT: paddd -{{[0-9]+}}(%rsp), %xmm3 # 16-byte Folded Reload 366; SSE2-NEXT: movdqa -{{[0-9]+}}(%rsp), %xmm9 # 16-byte Reload 367; SSE2-NEXT: paddd -{{[0-9]+}}(%rsp), %xmm9 # 16-byte Folded Reload 368; SSE2-NEXT: paddd -{{[0-9]+}}(%rsp), %xmm5 # 16-byte Folded Reload 369; SSE2-NEXT: movdqa -{{[0-9]+}}(%rsp), %xmm4 # 16-byte Reload 370; SSE2-NEXT: paddd -{{[0-9]+}}(%rsp), %xmm4 # 16-byte Folded Reload 371; SSE2-NEXT: paddd -{{[0-9]+}}(%rsp), %xmm2 # 16-byte Folded Reload 372; SSE2-NEXT: paddd (%rsp), %xmm11 # 16-byte Folded Reload 373; SSE2-NEXT: paddd {{[0-9]+}}(%rsp), %xmm6 # 16-byte Folded Reload 374; SSE2-NEXT: paddd {{[0-9]+}}(%rsp), %xmm12 # 16-byte Folded Reload 375; SSE2-NEXT: paddd {{[0-9]+}}(%rsp), %xmm1 # 16-byte Folded Reload 376; SSE2-NEXT: paddd {{[0-9]+}}(%rsp), %xmm13 # 16-byte Folded Reload 377; SSE2-NEXT: paddd {{[0-9]+}}(%rsp), %xmm7 # 16-byte Folded Reload 378; SSE2-NEXT: paddd {{[0-9]+}}(%rsp), %xmm14 # 16-byte Folded Reload 379; SSE2-NEXT: paddd {{[0-9]+}}(%rsp), %xmm15 # 16-byte Folded Reload 380; SSE2-NEXT: paddd {{[0-9]+}}(%rsp), %xmm10 # 16-byte Folded Reload 381; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [1,1,1,1] 382; SSE2-NEXT: paddd %xmm0, %xmm10 383; SSE2-NEXT: paddd %xmm0, %xmm15 384; SSE2-NEXT: paddd %xmm0, %xmm14 385; SSE2-NEXT: paddd %xmm0, %xmm7 386; SSE2-NEXT: paddd %xmm0, %xmm13 387; SSE2-NEXT: paddd %xmm0, %xmm1 388; SSE2-NEXT: paddd %xmm0, %xmm12 389; SSE2-NEXT: paddd %xmm0, %xmm6 390; SSE2-NEXT: paddd %xmm0, %xmm11 391; SSE2-NEXT: paddd %xmm0, %xmm2 392; SSE2-NEXT: paddd %xmm0, %xmm4 393; SSE2-NEXT: movdqa %xmm4, -{{[0-9]+}}(%rsp) # 16-byte Spill 394; SSE2-NEXT: paddd %xmm0, %xmm5 395; SSE2-NEXT: paddd %xmm0, %xmm9 396; SSE2-NEXT: paddd %xmm0, %xmm3 397; SSE2-NEXT: movdqa -{{[0-9]+}}(%rsp), %xmm4 # 16-byte Reload 398; SSE2-NEXT: paddd %xmm0, %xmm4 399; SSE2-NEXT: paddd %xmm0, %xmm8 400; SSE2-NEXT: psrld $1, %xmm15 401; SSE2-NEXT: psrld $1, %xmm10 402; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0] 403; SSE2-NEXT: pand %xmm0, %xmm10 404; SSE2-NEXT: pand %xmm0, %xmm15 405; SSE2-NEXT: packuswb %xmm10, %xmm15 406; SSE2-NEXT: psrld $1, %xmm7 407; SSE2-NEXT: psrld $1, %xmm14 408; SSE2-NEXT: pand %xmm0, %xmm14 409; SSE2-NEXT: pand %xmm0, %xmm7 410; SSE2-NEXT: packuswb %xmm14, %xmm7 411; SSE2-NEXT: packuswb %xmm7, %xmm15 412; SSE2-NEXT: psrld $1, %xmm1 413; SSE2-NEXT: psrld $1, %xmm13 414; SSE2-NEXT: pand %xmm0, %xmm13 415; SSE2-NEXT: pand %xmm0, %xmm1 416; SSE2-NEXT: packuswb %xmm13, %xmm1 417; SSE2-NEXT: psrld $1, %xmm6 418; SSE2-NEXT: psrld $1, %xmm12 419; SSE2-NEXT: pand %xmm0, %xmm12 420; SSE2-NEXT: pand %xmm0, %xmm6 421; SSE2-NEXT: packuswb %xmm12, %xmm6 422; SSE2-NEXT: packuswb %xmm6, %xmm1 423; SSE2-NEXT: psrld $1, %xmm2 424; SSE2-NEXT: psrld $1, %xmm11 425; SSE2-NEXT: pand %xmm0, %xmm11 426; SSE2-NEXT: pand %xmm0, %xmm2 427; SSE2-NEXT: packuswb %xmm11, %xmm2 428; SSE2-NEXT: psrld $1, %xmm5 429; SSE2-NEXT: movdqa -{{[0-9]+}}(%rsp), %xmm6 # 16-byte Reload 430; SSE2-NEXT: psrld $1, %xmm6 431; SSE2-NEXT: pand %xmm0, %xmm6 432; SSE2-NEXT: pand %xmm0, %xmm5 433; SSE2-NEXT: packuswb %xmm6, %xmm5 434; SSE2-NEXT: packuswb %xmm5, %xmm2 435; SSE2-NEXT: psrld $1, %xmm3 436; SSE2-NEXT: movdqa %xmm9, %xmm5 437; SSE2-NEXT: psrld $1, %xmm5 438; SSE2-NEXT: pand %xmm0, %xmm5 439; SSE2-NEXT: pand %xmm0, %xmm3 440; SSE2-NEXT: packuswb %xmm5, %xmm3 441; SSE2-NEXT: psrld $1, %xmm8 442; SSE2-NEXT: movdqa %xmm4, %xmm5 443; SSE2-NEXT: psrld $1, %xmm5 444; SSE2-NEXT: pand %xmm0, %xmm5 445; SSE2-NEXT: pand %xmm0, %xmm8 446; SSE2-NEXT: packuswb %xmm5, %xmm8 447; SSE2-NEXT: packuswb %xmm8, %xmm3 448; SSE2-NEXT: movdqu %xmm3, (%rax) 449; SSE2-NEXT: movdqu %xmm2, (%rax) 450; SSE2-NEXT: movdqu %xmm1, (%rax) 451; SSE2-NEXT: movdqu %xmm15, (%rax) 452; SSE2-NEXT: addq $152, %rsp 453; SSE2-NEXT: retq 454; 455; AVX2-LABEL: avg_v64i8: 456; AVX2: # BB#0: 457; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 458; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 459; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 460; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm3 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 461; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 462; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm5 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 463; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm6 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 464; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm7 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 465; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 466; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm9 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 467; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm10 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 468; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm11 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 469; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm12 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 470; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm13 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 471; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm14 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 472; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm15 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 473; AVX2-NEXT: vpaddd %ymm15, %ymm7, %ymm7 474; AVX2-NEXT: vpaddd %ymm14, %ymm6, %ymm6 475; AVX2-NEXT: vpaddd %ymm13, %ymm5, %ymm5 476; AVX2-NEXT: vpaddd %ymm12, %ymm4, %ymm4 477; AVX2-NEXT: vpaddd %ymm11, %ymm3, %ymm3 478; AVX2-NEXT: vpaddd %ymm10, %ymm2, %ymm2 479; AVX2-NEXT: vpaddd %ymm9, %ymm1, %ymm1 480; AVX2-NEXT: vpaddd %ymm8, %ymm0, %ymm0 481; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %ymm8 482; AVX2-NEXT: vpaddd %ymm8, %ymm0, %ymm9 483; AVX2-NEXT: vpaddd %ymm8, %ymm1, %ymm10 484; AVX2-NEXT: vpaddd %ymm8, %ymm2, %ymm2 485; AVX2-NEXT: vpaddd %ymm8, %ymm3, %ymm3 486; AVX2-NEXT: vpaddd %ymm8, %ymm4, %ymm4 487; AVX2-NEXT: vpaddd %ymm8, %ymm5, %ymm5 488; AVX2-NEXT: vpaddd %ymm8, %ymm6, %ymm1 489; AVX2-NEXT: vpaddd %ymm8, %ymm7, %ymm0 490; AVX2-NEXT: vpsrld $1, %ymm0, %ymm11 491; AVX2-NEXT: vpsrld $1, %ymm1, %ymm12 492; AVX2-NEXT: vpsrld $1, %ymm5, %ymm5 493; AVX2-NEXT: vpsrld $1, %ymm4, %ymm4 494; AVX2-NEXT: vpsrld $1, %ymm3, %ymm6 495; AVX2-NEXT: vpsrld $1, %ymm2, %ymm7 496; AVX2-NEXT: vpsrld $1, %ymm10, %ymm8 497; AVX2-NEXT: vpsrld $1, %ymm9, %ymm3 498; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [0,1,4,5,8,9,12,13,128,128,128,128,128,128,128,128,0,1,4,5,8,9,12,13,128,128,128,128,128,128,128,128] 499; AVX2-NEXT: vpshufb %ymm2, %ymm3, %ymm3 500; AVX2-NEXT: vpermq {{.*#+}} ymm9 = ymm3[0,2,2,3] 501; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u> 502; AVX2-NEXT: vpshufb %xmm3, %xmm9, %xmm0 503; AVX2-NEXT: vpshufb %ymm2, %ymm8, %ymm8 504; AVX2-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,2,3] 505; AVX2-NEXT: vpshufb %xmm3, %xmm8, %xmm1 506; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] 507; AVX2-NEXT: vpshufb %ymm2, %ymm7, %ymm1 508; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3] 509; AVX2-NEXT: vpshufb %xmm3, %xmm1, %xmm1 510; AVX2-NEXT: vpshufb %ymm2, %ymm6, %ymm6 511; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm6[0,2,2,3] 512; AVX2-NEXT: vpshufb %xmm3, %xmm6, %xmm6 513; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm6[0],xmm1[0] 514; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 515; AVX2-NEXT: vpshufb %ymm2, %ymm4, %ymm1 516; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3] 517; AVX2-NEXT: vpshufb %xmm3, %xmm1, %xmm1 518; AVX2-NEXT: vpshufb %ymm2, %ymm5, %ymm4 519; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,2,3] 520; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm4 521; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm4[0],xmm1[0] 522; AVX2-NEXT: vpshufb %ymm2, %ymm12, %ymm4 523; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,2,3] 524; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm4 525; AVX2-NEXT: vpshufb %ymm2, %ymm11, %ymm2 526; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3] 527; AVX2-NEXT: vpshufb %xmm3, %xmm2, %xmm2 528; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0] 529; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1 530; AVX2-NEXT: vmovdqu %ymm1, (%rax) 531; AVX2-NEXT: vmovdqu %ymm0, (%rax) 532; AVX2-NEXT: vzeroupper 533; AVX2-NEXT: retq 534; 535; AVX512F-LABEL: avg_v64i8: 536; AVX512F: # BB#0: 537; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero 538; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero 539; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero 540; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm3 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero 541; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero 542; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm5 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero 543; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm6 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero 544; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm7 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero 545; AVX512F-NEXT: vpaddd %zmm7, %zmm3, %zmm3 546; AVX512F-NEXT: vpaddd %zmm6, %zmm2, %zmm2 547; AVX512F-NEXT: vpaddd %zmm5, %zmm1, %zmm1 548; AVX512F-NEXT: vpaddd %zmm4, %zmm0, %zmm0 549; AVX512F-NEXT: vpbroadcastd {{.*}}(%rip), %zmm4 550; AVX512F-NEXT: vpaddd %zmm4, %zmm0, %zmm0 551; AVX512F-NEXT: vpaddd %zmm4, %zmm1, %zmm1 552; AVX512F-NEXT: vpaddd %zmm4, %zmm2, %zmm2 553; AVX512F-NEXT: vpaddd %zmm4, %zmm3, %zmm3 554; AVX512F-NEXT: vpsrld $1, %zmm3, %zmm3 555; AVX512F-NEXT: vpsrld $1, %zmm2, %zmm2 556; AVX512F-NEXT: vpsrld $1, %zmm1, %zmm1 557; AVX512F-NEXT: vpsrld $1, %zmm0, %zmm0 558; AVX512F-NEXT: vpmovdb %zmm0, %xmm0 559; AVX512F-NEXT: vpmovdb %zmm1, %xmm1 560; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 561; AVX512F-NEXT: vpmovdb %zmm2, %xmm1 562; AVX512F-NEXT: vpmovdb %zmm3, %xmm2 563; AVX512F-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 564; AVX512F-NEXT: vmovdqu %ymm1, (%rax) 565; AVX512F-NEXT: vmovdqu %ymm0, (%rax) 566; AVX512F-NEXT: retq 567; 568; AVX512BW-LABEL: avg_v64i8: 569; AVX512BW: # BB#0: 570; AVX512BW-NEXT: vmovdqu8 (%rsi), %zmm0 571; AVX512BW-NEXT: vpavgb (%rdi), %zmm0, %zmm0 572; AVX512BW-NEXT: vmovdqu8 %zmm0, (%rax) 573; AVX512BW-NEXT: retq 574 %1 = load <64 x i8>, <64 x i8>* %a 575 %2 = load <64 x i8>, <64 x i8>* %b 576 %3 = zext <64 x i8> %1 to <64 x i32> 577 %4 = zext <64 x i8> %2 to <64 x i32> 578 %5 = add nuw nsw <64 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 579 %6 = add nuw nsw <64 x i32> %5, %4 580 %7 = lshr <64 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 581 %8 = trunc <64 x i32> %7 to <64 x i8> 582 store <64 x i8> %8, <64 x i8>* undef, align 4 583 ret void 584} 585 586define void @avg_v4i16(<4 x i16>* %a, <4 x i16>* %b) { 587; SSE2-LABEL: avg_v4i16: 588; SSE2: # BB#0: 589; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero 590; SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero 591; SSE2-NEXT: pavgw %xmm0, %xmm1 592; SSE2-NEXT: movq %xmm1, (%rax) 593; SSE2-NEXT: retq 594; 595; AVX2-LABEL: avg_v4i16: 596; AVX2: # BB#0: 597; AVX2-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 598; AVX2-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero 599; AVX2-NEXT: vpavgw %xmm0, %xmm1, %xmm0 600; AVX2-NEXT: vmovq %xmm0, (%rax) 601; AVX2-NEXT: retq 602; 603; AVX512F-LABEL: avg_v4i16: 604; AVX512F: # BB#0: 605; AVX512F-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 606; AVX512F-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero 607; AVX512F-NEXT: vpavgw %xmm0, %xmm1, %xmm0 608; AVX512F-NEXT: vmovq %xmm0, (%rax) 609; AVX512F-NEXT: retq 610; 611; AVX512BW-LABEL: avg_v4i16: 612; AVX512BW: # BB#0: 613; AVX512BW-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 614; AVX512BW-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero 615; AVX512BW-NEXT: vpavgw %xmm0, %xmm1, %xmm0 616; AVX512BW-NEXT: vmovq %xmm0, (%rax) 617; AVX512BW-NEXT: retq 618 %1 = load <4 x i16>, <4 x i16>* %a 619 %2 = load <4 x i16>, <4 x i16>* %b 620 %3 = zext <4 x i16> %1 to <4 x i32> 621 %4 = zext <4 x i16> %2 to <4 x i32> 622 %5 = add nuw nsw <4 x i32> %3, <i32 1, i32 1, i32 1, i32 1> 623 %6 = add nuw nsw <4 x i32> %5, %4 624 %7 = lshr <4 x i32> %6, <i32 1, i32 1, i32 1, i32 1> 625 %8 = trunc <4 x i32> %7 to <4 x i16> 626 store <4 x i16> %8, <4 x i16>* undef, align 4 627 ret void 628} 629 630define void @avg_v8i16(<8 x i16>* %a, <8 x i16>* %b) { 631; SSE2-LABEL: avg_v8i16: 632; SSE2: # BB#0: 633; SSE2-NEXT: movdqa (%rsi), %xmm0 634; SSE2-NEXT: pavgw (%rdi), %xmm0 635; SSE2-NEXT: movdqu %xmm0, (%rax) 636; SSE2-NEXT: retq 637; 638; AVX2-LABEL: avg_v8i16: 639; AVX2: # BB#0: 640; AVX2-NEXT: vmovdqa (%rsi), %xmm0 641; AVX2-NEXT: vpavgw (%rdi), %xmm0, %xmm0 642; AVX2-NEXT: vmovdqu %xmm0, (%rax) 643; AVX2-NEXT: retq 644; 645; AVX512F-LABEL: avg_v8i16: 646; AVX512F: # BB#0: 647; AVX512F-NEXT: vmovdqa (%rsi), %xmm0 648; AVX512F-NEXT: vpavgw (%rdi), %xmm0, %xmm0 649; AVX512F-NEXT: vmovdqu %xmm0, (%rax) 650; AVX512F-NEXT: retq 651; 652; AVX512BW-LABEL: avg_v8i16: 653; AVX512BW: # BB#0: 654; AVX512BW-NEXT: vmovdqa (%rsi), %xmm0 655; AVX512BW-NEXT: vpavgw (%rdi), %xmm0, %xmm0 656; AVX512BW-NEXT: vmovdqu %xmm0, (%rax) 657; AVX512BW-NEXT: retq 658 %1 = load <8 x i16>, <8 x i16>* %a 659 %2 = load <8 x i16>, <8 x i16>* %b 660 %3 = zext <8 x i16> %1 to <8 x i32> 661 %4 = zext <8 x i16> %2 to <8 x i32> 662 %5 = add nuw nsw <8 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 663 %6 = add nuw nsw <8 x i32> %5, %4 664 %7 = lshr <8 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 665 %8 = trunc <8 x i32> %7 to <8 x i16> 666 store <8 x i16> %8, <8 x i16>* undef, align 4 667 ret void 668} 669 670define void @avg_v16i16(<16 x i16>* %a, <16 x i16>* %b) { 671; SSE2-LABEL: avg_v16i16: 672; SSE2: # BB#0: 673; SSE2-NEXT: movdqa (%rdi), %xmm4 674; SSE2-NEXT: movdqa 16(%rdi), %xmm5 675; SSE2-NEXT: movdqa (%rsi), %xmm0 676; SSE2-NEXT: movdqa 16(%rsi), %xmm1 677; SSE2-NEXT: pxor %xmm6, %xmm6 678; SSE2-NEXT: movdqa %xmm4, %xmm8 679; SSE2-NEXT: punpckhwd {{.*#+}} xmm8 = xmm8[4],xmm6[4],xmm8[5],xmm6[5],xmm8[6],xmm6[6],xmm8[7],xmm6[7] 680; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3] 681; SSE2-NEXT: movdqa %xmm5, %xmm7 682; SSE2-NEXT: punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7] 683; SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3] 684; SSE2-NEXT: movdqa %xmm0, %xmm3 685; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm6[4],xmm3[5],xmm6[5],xmm3[6],xmm6[6],xmm3[7],xmm6[7] 686; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3] 687; SSE2-NEXT: movdqa %xmm1, %xmm2 688; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7] 689; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1],xmm1[2],xmm6[2],xmm1[3],xmm6[3] 690; SSE2-NEXT: paddd %xmm5, %xmm1 691; SSE2-NEXT: paddd %xmm7, %xmm2 692; SSE2-NEXT: paddd %xmm4, %xmm0 693; SSE2-NEXT: paddd %xmm8, %xmm3 694; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [1,1,1,1] 695; SSE2-NEXT: paddd %xmm4, %xmm3 696; SSE2-NEXT: paddd %xmm4, %xmm0 697; SSE2-NEXT: paddd %xmm4, %xmm2 698; SSE2-NEXT: paddd %xmm4, %xmm1 699; SSE2-NEXT: psrld $1, %xmm1 700; SSE2-NEXT: psrld $1, %xmm2 701; SSE2-NEXT: psrld $1, %xmm0 702; SSE2-NEXT: psrld $1, %xmm3 703; SSE2-NEXT: pslld $16, %xmm3 704; SSE2-NEXT: psrad $16, %xmm3 705; SSE2-NEXT: pslld $16, %xmm0 706; SSE2-NEXT: psrad $16, %xmm0 707; SSE2-NEXT: packssdw %xmm3, %xmm0 708; SSE2-NEXT: pslld $16, %xmm2 709; SSE2-NEXT: psrad $16, %xmm2 710; SSE2-NEXT: pslld $16, %xmm1 711; SSE2-NEXT: psrad $16, %xmm1 712; SSE2-NEXT: packssdw %xmm2, %xmm1 713; SSE2-NEXT: movdqu %xmm1, (%rax) 714; SSE2-NEXT: movdqu %xmm0, (%rax) 715; SSE2-NEXT: retq 716; 717; AVX2-LABEL: avg_v16i16: 718; AVX2: # BB#0: 719; AVX2-NEXT: vmovdqa (%rsi), %ymm0 720; AVX2-NEXT: vpavgw (%rdi), %ymm0, %ymm0 721; AVX2-NEXT: vmovdqu %ymm0, (%rax) 722; AVX2-NEXT: vzeroupper 723; AVX2-NEXT: retq 724; 725; AVX512F-LABEL: avg_v16i16: 726; AVX512F: # BB#0: 727; AVX512F-NEXT: vmovdqa (%rsi), %ymm0 728; AVX512F-NEXT: vpavgw (%rdi), %ymm0, %ymm0 729; AVX512F-NEXT: vmovdqu %ymm0, (%rax) 730; AVX512F-NEXT: retq 731; 732; AVX512BW-LABEL: avg_v16i16: 733; AVX512BW: # BB#0: 734; AVX512BW-NEXT: vmovdqa (%rsi), %ymm0 735; AVX512BW-NEXT: vpavgw (%rdi), %ymm0, %ymm0 736; AVX512BW-NEXT: vmovdqu %ymm0, (%rax) 737; AVX512BW-NEXT: retq 738 %1 = load <16 x i16>, <16 x i16>* %a 739 %2 = load <16 x i16>, <16 x i16>* %b 740 %3 = zext <16 x i16> %1 to <16 x i32> 741 %4 = zext <16 x i16> %2 to <16 x i32> 742 %5 = add nuw nsw <16 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 743 %6 = add nuw nsw <16 x i32> %5, %4 744 %7 = lshr <16 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 745 %8 = trunc <16 x i32> %7 to <16 x i16> 746 store <16 x i16> %8, <16 x i16>* undef, align 4 747 ret void 748} 749 750define void @avg_v32i16(<32 x i16>* %a, <32 x i16>* %b) { 751; SSE2-LABEL: avg_v32i16: 752; SSE2: # BB#0: 753; SSE2-NEXT: movdqa (%rdi), %xmm10 754; SSE2-NEXT: movdqa 16(%rdi), %xmm9 755; SSE2-NEXT: movdqa 32(%rdi), %xmm11 756; SSE2-NEXT: movdqa 48(%rdi), %xmm8 757; SSE2-NEXT: movdqa (%rsi), %xmm14 758; SSE2-NEXT: movdqa 16(%rsi), %xmm1 759; SSE2-NEXT: movdqa 32(%rsi), %xmm2 760; SSE2-NEXT: movdqa 48(%rsi), %xmm3 761; SSE2-NEXT: pxor %xmm0, %xmm0 762; SSE2-NEXT: movdqa %xmm10, %xmm4 763; SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7] 764; SSE2-NEXT: movdqa %xmm4, -{{[0-9]+}}(%rsp) # 16-byte Spill 765; SSE2-NEXT: punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm0[0],xmm10[1],xmm0[1],xmm10[2],xmm0[2],xmm10[3],xmm0[3] 766; SSE2-NEXT: movdqa %xmm9, %xmm12 767; SSE2-NEXT: punpckhwd {{.*#+}} xmm12 = xmm12[4],xmm0[4],xmm12[5],xmm0[5],xmm12[6],xmm0[6],xmm12[7],xmm0[7] 768; SSE2-NEXT: punpcklwd {{.*#+}} xmm9 = xmm9[0],xmm0[0],xmm9[1],xmm0[1],xmm9[2],xmm0[2],xmm9[3],xmm0[3] 769; SSE2-NEXT: movdqa %xmm11, %xmm15 770; SSE2-NEXT: punpckhwd {{.*#+}} xmm15 = xmm15[4],xmm0[4],xmm15[5],xmm0[5],xmm15[6],xmm0[6],xmm15[7],xmm0[7] 771; SSE2-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm0[0],xmm11[1],xmm0[1],xmm11[2],xmm0[2],xmm11[3],xmm0[3] 772; SSE2-NEXT: movdqa %xmm8, %xmm13 773; SSE2-NEXT: punpckhwd {{.*#+}} xmm13 = xmm13[4],xmm0[4],xmm13[5],xmm0[5],xmm13[6],xmm0[6],xmm13[7],xmm0[7] 774; SSE2-NEXT: punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3] 775; SSE2-NEXT: movdqa %xmm14, %xmm7 776; SSE2-NEXT: punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7] 777; SSE2-NEXT: punpcklwd {{.*#+}} xmm14 = xmm14[0],xmm0[0],xmm14[1],xmm0[1],xmm14[2],xmm0[2],xmm14[3],xmm0[3] 778; SSE2-NEXT: movdqa %xmm1, %xmm6 779; SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7] 780; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] 781; SSE2-NEXT: movdqa %xmm2, %xmm5 782; SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7] 783; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3] 784; SSE2-NEXT: movdqa %xmm3, %xmm4 785; SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7] 786; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3] 787; SSE2-NEXT: paddd %xmm8, %xmm3 788; SSE2-NEXT: paddd %xmm13, %xmm4 789; SSE2-NEXT: paddd %xmm11, %xmm2 790; SSE2-NEXT: paddd %xmm15, %xmm5 791; SSE2-NEXT: paddd %xmm9, %xmm1 792; SSE2-NEXT: paddd %xmm12, %xmm6 793; SSE2-NEXT: paddd %xmm10, %xmm14 794; SSE2-NEXT: paddd -{{[0-9]+}}(%rsp), %xmm7 # 16-byte Folded Reload 795; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [1,1,1,1] 796; SSE2-NEXT: paddd %xmm0, %xmm7 797; SSE2-NEXT: paddd %xmm0, %xmm14 798; SSE2-NEXT: paddd %xmm0, %xmm6 799; SSE2-NEXT: paddd %xmm0, %xmm1 800; SSE2-NEXT: paddd %xmm0, %xmm5 801; SSE2-NEXT: paddd %xmm0, %xmm2 802; SSE2-NEXT: paddd %xmm0, %xmm4 803; SSE2-NEXT: paddd %xmm0, %xmm3 804; SSE2-NEXT: psrld $1, %xmm14 805; SSE2-NEXT: psrld $1, %xmm7 806; SSE2-NEXT: pslld $16, %xmm7 807; SSE2-NEXT: psrad $16, %xmm7 808; SSE2-NEXT: pslld $16, %xmm14 809; SSE2-NEXT: psrad $16, %xmm14 810; SSE2-NEXT: packssdw %xmm7, %xmm14 811; SSE2-NEXT: psrld $1, %xmm1 812; SSE2-NEXT: psrld $1, %xmm6 813; SSE2-NEXT: pslld $16, %xmm6 814; SSE2-NEXT: psrad $16, %xmm6 815; SSE2-NEXT: pslld $16, %xmm1 816; SSE2-NEXT: psrad $16, %xmm1 817; SSE2-NEXT: packssdw %xmm6, %xmm1 818; SSE2-NEXT: psrld $1, %xmm2 819; SSE2-NEXT: psrld $1, %xmm5 820; SSE2-NEXT: pslld $16, %xmm5 821; SSE2-NEXT: psrad $16, %xmm5 822; SSE2-NEXT: pslld $16, %xmm2 823; SSE2-NEXT: psrad $16, %xmm2 824; SSE2-NEXT: packssdw %xmm5, %xmm2 825; SSE2-NEXT: psrld $1, %xmm3 826; SSE2-NEXT: psrld $1, %xmm4 827; SSE2-NEXT: pslld $16, %xmm4 828; SSE2-NEXT: psrad $16, %xmm4 829; SSE2-NEXT: pslld $16, %xmm3 830; SSE2-NEXT: psrad $16, %xmm3 831; SSE2-NEXT: packssdw %xmm4, %xmm3 832; SSE2-NEXT: movdqu %xmm3, (%rax) 833; SSE2-NEXT: movdqu %xmm2, (%rax) 834; SSE2-NEXT: movdqu %xmm1, (%rax) 835; SSE2-NEXT: movdqu %xmm14, (%rax) 836; SSE2-NEXT: retq 837; 838; AVX2-LABEL: avg_v32i16: 839; AVX2: # BB#0: 840; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 841; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 842; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 843; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 844; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 845; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm5 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 846; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm6 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 847; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm7 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 848; AVX2-NEXT: vpaddd %ymm7, %ymm3, %ymm3 849; AVX2-NEXT: vpaddd %ymm6, %ymm2, %ymm2 850; AVX2-NEXT: vpaddd %ymm5, %ymm1, %ymm1 851; AVX2-NEXT: vpaddd %ymm4, %ymm0, %ymm0 852; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %ymm4 853; AVX2-NEXT: vpaddd %ymm4, %ymm0, %ymm0 854; AVX2-NEXT: vpaddd %ymm4, %ymm1, %ymm1 855; AVX2-NEXT: vpaddd %ymm4, %ymm2, %ymm2 856; AVX2-NEXT: vpaddd %ymm4, %ymm3, %ymm3 857; AVX2-NEXT: vpsrld $1, %ymm3, %ymm3 858; AVX2-NEXT: vpsrld $1, %ymm2, %ymm2 859; AVX2-NEXT: vpsrld $1, %ymm1, %ymm1 860; AVX2-NEXT: vpsrld $1, %ymm0, %ymm0 861; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,4,5,8,9,12,13,128,128,128,128,128,128,128,128,0,1,4,5,8,9,12,13,128,128,128,128,128,128,128,128] 862; AVX2-NEXT: vpshufb %ymm4, %ymm0, %ymm0 863; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3] 864; AVX2-NEXT: vpshufb %ymm4, %ymm1, %ymm1 865; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3] 866; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 867; AVX2-NEXT: vpshufb %ymm4, %ymm2, %ymm1 868; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3] 869; AVX2-NEXT: vpshufb %ymm4, %ymm3, %ymm2 870; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3] 871; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 872; AVX2-NEXT: vmovdqu %ymm1, (%rax) 873; AVX2-NEXT: vmovdqu %ymm0, (%rax) 874; AVX2-NEXT: vzeroupper 875; AVX2-NEXT: retq 876; 877; AVX512F-LABEL: avg_v32i16: 878; AVX512F: # BB#0: 879; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero 880; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero 881; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero 882; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero 883; AVX512F-NEXT: vpaddd %zmm3, %zmm1, %zmm1 884; AVX512F-NEXT: vpaddd %zmm2, %zmm0, %zmm0 885; AVX512F-NEXT: vpbroadcastd {{.*}}(%rip), %zmm2 886; AVX512F-NEXT: vpaddd %zmm2, %zmm0, %zmm0 887; AVX512F-NEXT: vpaddd %zmm2, %zmm1, %zmm1 888; AVX512F-NEXT: vpsrld $1, %zmm1, %zmm1 889; AVX512F-NEXT: vpsrld $1, %zmm0, %zmm0 890; AVX512F-NEXT: vpmovdw %zmm0, (%rax) 891; AVX512F-NEXT: vpmovdw %zmm1, (%rax) 892; AVX512F-NEXT: retq 893; 894; AVX512BW-LABEL: avg_v32i16: 895; AVX512BW: # BB#0: 896; AVX512BW-NEXT: vmovdqu16 (%rsi), %zmm0 897; AVX512BW-NEXT: vpavgw (%rdi), %zmm0, %zmm0 898; AVX512BW-NEXT: vmovdqu16 %zmm0, (%rax) 899; AVX512BW-NEXT: retq 900 %1 = load <32 x i16>, <32 x i16>* %a 901 %2 = load <32 x i16>, <32 x i16>* %b 902 %3 = zext <32 x i16> %1 to <32 x i32> 903 %4 = zext <32 x i16> %2 to <32 x i32> 904 %5 = add nuw nsw <32 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 905 %6 = add nuw nsw <32 x i32> %5, %4 906 %7 = lshr <32 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 907 %8 = trunc <32 x i32> %7 to <32 x i16> 908 store <32 x i16> %8, <32 x i16>* undef, align 4 909 ret void 910} 911 912define void @avg_v4i8_2(<4 x i8>* %a, <4 x i8>* %b) { 913; SSE2-LABEL: avg_v4i8_2: 914; SSE2: # BB#0: 915; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero 916; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero 917; SSE2-NEXT: pavgb %xmm0, %xmm1 918; SSE2-NEXT: movd %xmm1, (%rax) 919; SSE2-NEXT: retq 920; 921; AVX2-LABEL: avg_v4i8_2: 922; AVX2: # BB#0: 923; AVX2-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero 924; AVX2-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero 925; AVX2-NEXT: vpavgb %xmm1, %xmm0, %xmm0 926; AVX2-NEXT: vmovd %xmm0, (%rax) 927; AVX2-NEXT: retq 928; 929; AVX512F-LABEL: avg_v4i8_2: 930; AVX512F: # BB#0: 931; AVX512F-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero 932; AVX512F-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero 933; AVX512F-NEXT: vpavgb %xmm1, %xmm0, %xmm0 934; AVX512F-NEXT: vmovd %xmm0, (%rax) 935; AVX512F-NEXT: retq 936; 937; AVX512BW-LABEL: avg_v4i8_2: 938; AVX512BW: # BB#0: 939; AVX512BW-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero 940; AVX512BW-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero 941; AVX512BW-NEXT: vpavgb %xmm1, %xmm0, %xmm0 942; AVX512BW-NEXT: vmovd %xmm0, (%rax) 943; AVX512BW-NEXT: retq 944 %1 = load <4 x i8>, <4 x i8>* %a 945 %2 = load <4 x i8>, <4 x i8>* %b 946 %3 = zext <4 x i8> %1 to <4 x i32> 947 %4 = zext <4 x i8> %2 to <4 x i32> 948 %5 = add nuw nsw <4 x i32> %3, %4 949 %6 = add nuw nsw <4 x i32> %5, <i32 1, i32 1, i32 1, i32 1> 950 %7 = lshr <4 x i32> %6, <i32 1, i32 1, i32 1, i32 1> 951 %8 = trunc <4 x i32> %7 to <4 x i8> 952 store <4 x i8> %8, <4 x i8>* undef, align 4 953 ret void 954} 955 956define void @avg_v8i8_2(<8 x i8>* %a, <8 x i8>* %b) { 957; SSE2-LABEL: avg_v8i8_2: 958; SSE2: # BB#0: 959; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero 960; SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero 961; SSE2-NEXT: pavgb %xmm0, %xmm1 962; SSE2-NEXT: movq %xmm1, (%rax) 963; SSE2-NEXT: retq 964; 965; AVX2-LABEL: avg_v8i8_2: 966; AVX2: # BB#0: 967; AVX2-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 968; AVX2-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero 969; AVX2-NEXT: vpavgb %xmm1, %xmm0, %xmm0 970; AVX2-NEXT: vmovq %xmm0, (%rax) 971; AVX2-NEXT: retq 972; 973; AVX512F-LABEL: avg_v8i8_2: 974; AVX512F: # BB#0: 975; AVX512F-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 976; AVX512F-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero 977; AVX512F-NEXT: vpavgb %xmm1, %xmm0, %xmm0 978; AVX512F-NEXT: vmovq %xmm0, (%rax) 979; AVX512F-NEXT: retq 980; 981; AVX512BW-LABEL: avg_v8i8_2: 982; AVX512BW: # BB#0: 983; AVX512BW-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 984; AVX512BW-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero 985; AVX512BW-NEXT: vpavgb %xmm1, %xmm0, %xmm0 986; AVX512BW-NEXT: vmovq %xmm0, (%rax) 987; AVX512BW-NEXT: retq 988 %1 = load <8 x i8>, <8 x i8>* %a 989 %2 = load <8 x i8>, <8 x i8>* %b 990 %3 = zext <8 x i8> %1 to <8 x i32> 991 %4 = zext <8 x i8> %2 to <8 x i32> 992 %5 = add nuw nsw <8 x i32> %3, %4 993 %6 = add nuw nsw <8 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 994 %7 = lshr <8 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 995 %8 = trunc <8 x i32> %7 to <8 x i8> 996 store <8 x i8> %8, <8 x i8>* undef, align 4 997 ret void 998} 999 1000define void @avg_v16i8_2(<16 x i8>* %a, <16 x i8>* %b) { 1001; SSE2-LABEL: avg_v16i8_2: 1002; SSE2: # BB#0: 1003; SSE2-NEXT: movdqa (%rdi), %xmm0 1004; SSE2-NEXT: pavgb (%rsi), %xmm0 1005; SSE2-NEXT: movdqu %xmm0, (%rax) 1006; SSE2-NEXT: retq 1007; 1008; AVX2-LABEL: avg_v16i8_2: 1009; AVX2: # BB#0: 1010; AVX2-NEXT: vmovdqa (%rdi), %xmm0 1011; AVX2-NEXT: vpavgb (%rsi), %xmm0, %xmm0 1012; AVX2-NEXT: vmovdqu %xmm0, (%rax) 1013; AVX2-NEXT: retq 1014; 1015; AVX512F-LABEL: avg_v16i8_2: 1016; AVX512F: # BB#0: 1017; AVX512F-NEXT: vmovdqa (%rdi), %xmm0 1018; AVX512F-NEXT: vpavgb (%rsi), %xmm0, %xmm0 1019; AVX512F-NEXT: vmovdqu %xmm0, (%rax) 1020; AVX512F-NEXT: retq 1021; 1022; AVX512BW-LABEL: avg_v16i8_2: 1023; AVX512BW: # BB#0: 1024; AVX512BW-NEXT: vmovdqa (%rdi), %xmm0 1025; AVX512BW-NEXT: vpavgb (%rsi), %xmm0, %xmm0 1026; AVX512BW-NEXT: vmovdqu %xmm0, (%rax) 1027; AVX512BW-NEXT: retq 1028 %1 = load <16 x i8>, <16 x i8>* %a 1029 %2 = load <16 x i8>, <16 x i8>* %b 1030 %3 = zext <16 x i8> %1 to <16 x i32> 1031 %4 = zext <16 x i8> %2 to <16 x i32> 1032 %5 = add nuw nsw <16 x i32> %3, %4 1033 %6 = add nuw nsw <16 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1034 %7 = lshr <16 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1035 %8 = trunc <16 x i32> %7 to <16 x i8> 1036 store <16 x i8> %8, <16 x i8>* undef, align 4 1037 ret void 1038} 1039 1040define void @avg_v32i8_2(<32 x i8>* %a, <32 x i8>* %b) { 1041; SSE2-LABEL: avg_v32i8_2: 1042; SSE2: # BB#0: 1043; SSE2-NEXT: movdqa (%rdi), %xmm8 1044; SSE2-NEXT: movdqa 16(%rdi), %xmm11 1045; SSE2-NEXT: movdqa (%rsi), %xmm0 1046; SSE2-NEXT: movdqa 16(%rsi), %xmm1 1047; SSE2-NEXT: pxor %xmm4, %xmm4 1048; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm8[2,3,0,1] 1049; SSE2-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1],xmm8[2],xmm4[2],xmm8[3],xmm4[3],xmm8[4],xmm4[4],xmm8[5],xmm4[5],xmm8[6],xmm4[6],xmm8[7],xmm4[7] 1050; SSE2-NEXT: movdqa %xmm8, %xmm2 1051; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7] 1052; SSE2-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp) # 16-byte Spill 1053; SSE2-NEXT: punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1],xmm8[2],xmm4[2],xmm8[3],xmm4[3] 1054; SSE2-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3],xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7] 1055; SSE2-NEXT: movdqa %xmm10, %xmm12 1056; SSE2-NEXT: punpckhwd {{.*#+}} xmm12 = xmm12[4],xmm4[4],xmm12[5],xmm4[5],xmm12[6],xmm4[6],xmm12[7],xmm4[7] 1057; SSE2-NEXT: punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3] 1058; SSE2-NEXT: pshufd {{.*#+}} xmm15 = xmm11[2,3,0,1] 1059; SSE2-NEXT: punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1],xmm11[2],xmm4[2],xmm11[3],xmm4[3],xmm11[4],xmm4[4],xmm11[5],xmm4[5],xmm11[6],xmm4[6],xmm11[7],xmm4[7] 1060; SSE2-NEXT: movdqa %xmm11, %xmm14 1061; SSE2-NEXT: punpckhwd {{.*#+}} xmm14 = xmm14[4],xmm4[4],xmm14[5],xmm4[5],xmm14[6],xmm4[6],xmm14[7],xmm4[7] 1062; SSE2-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1],xmm11[2],xmm4[2],xmm11[3],xmm4[3] 1063; SSE2-NEXT: punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm4[0],xmm15[1],xmm4[1],xmm15[2],xmm4[2],xmm15[3],xmm4[3],xmm15[4],xmm4[4],xmm15[5],xmm4[5],xmm15[6],xmm4[6],xmm15[7],xmm4[7] 1064; SSE2-NEXT: movdqa %xmm15, %xmm9 1065; SSE2-NEXT: punpckhwd {{.*#+}} xmm9 = xmm9[4],xmm4[4],xmm9[5],xmm4[5],xmm9[6],xmm4[6],xmm9[7],xmm4[7] 1066; SSE2-NEXT: punpcklwd {{.*#+}} xmm15 = xmm15[0],xmm4[0],xmm15[1],xmm4[1],xmm15[2],xmm4[2],xmm15[3],xmm4[3] 1067; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1] 1068; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7] 1069; SSE2-NEXT: movdqa %xmm0, %xmm7 1070; SSE2-NEXT: punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm4[4],xmm7[5],xmm4[5],xmm7[6],xmm4[6],xmm7[7],xmm4[7] 1071; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3] 1072; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7] 1073; SSE2-NEXT: movdqa %xmm3, %xmm6 1074; SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7] 1075; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3] 1076; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1] 1077; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] 1078; SSE2-NEXT: movdqa %xmm1, %xmm5 1079; SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7] 1080; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3] 1081; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7] 1082; SSE2-NEXT: movdqa %xmm2, %xmm13 1083; SSE2-NEXT: punpckhwd {{.*#+}} xmm13 = xmm13[4],xmm4[4],xmm13[5],xmm4[5],xmm13[6],xmm4[6],xmm13[7],xmm4[7] 1084; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3] 1085; SSE2-NEXT: paddd %xmm15, %xmm2 1086; SSE2-NEXT: paddd %xmm9, %xmm13 1087; SSE2-NEXT: paddd %xmm11, %xmm1 1088; SSE2-NEXT: paddd %xmm14, %xmm5 1089; SSE2-NEXT: paddd %xmm10, %xmm3 1090; SSE2-NEXT: paddd %xmm12, %xmm6 1091; SSE2-NEXT: paddd %xmm8, %xmm0 1092; SSE2-NEXT: paddd -{{[0-9]+}}(%rsp), %xmm7 # 16-byte Folded Reload 1093; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [1,1,1,1] 1094; SSE2-NEXT: paddd %xmm4, %xmm7 1095; SSE2-NEXT: paddd %xmm4, %xmm0 1096; SSE2-NEXT: paddd %xmm4, %xmm6 1097; SSE2-NEXT: paddd %xmm4, %xmm3 1098; SSE2-NEXT: paddd %xmm4, %xmm5 1099; SSE2-NEXT: paddd %xmm4, %xmm1 1100; SSE2-NEXT: paddd %xmm4, %xmm13 1101; SSE2-NEXT: paddd %xmm4, %xmm2 1102; SSE2-NEXT: psrld $1, %xmm0 1103; SSE2-NEXT: psrld $1, %xmm7 1104; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0] 1105; SSE2-NEXT: pand %xmm4, %xmm7 1106; SSE2-NEXT: pand %xmm4, %xmm0 1107; SSE2-NEXT: packuswb %xmm7, %xmm0 1108; SSE2-NEXT: psrld $1, %xmm3 1109; SSE2-NEXT: psrld $1, %xmm6 1110; SSE2-NEXT: pand %xmm4, %xmm6 1111; SSE2-NEXT: pand %xmm4, %xmm3 1112; SSE2-NEXT: packuswb %xmm6, %xmm3 1113; SSE2-NEXT: packuswb %xmm3, %xmm0 1114; SSE2-NEXT: psrld $1, %xmm1 1115; SSE2-NEXT: psrld $1, %xmm5 1116; SSE2-NEXT: pand %xmm4, %xmm5 1117; SSE2-NEXT: pand %xmm4, %xmm1 1118; SSE2-NEXT: packuswb %xmm5, %xmm1 1119; SSE2-NEXT: psrld $1, %xmm2 1120; SSE2-NEXT: psrld $1, %xmm13 1121; SSE2-NEXT: pand %xmm4, %xmm13 1122; SSE2-NEXT: pand %xmm4, %xmm2 1123; SSE2-NEXT: packuswb %xmm13, %xmm2 1124; SSE2-NEXT: packuswb %xmm2, %xmm1 1125; SSE2-NEXT: movdqu %xmm1, (%rax) 1126; SSE2-NEXT: movdqu %xmm0, (%rax) 1127; SSE2-NEXT: retq 1128; 1129; AVX2-LABEL: avg_v32i8_2: 1130; AVX2: # BB#0: 1131; AVX2-NEXT: vmovdqa (%rdi), %ymm0 1132; AVX2-NEXT: vpavgb (%rsi), %ymm0, %ymm0 1133; AVX2-NEXT: vmovdqu %ymm0, (%rax) 1134; AVX2-NEXT: vzeroupper 1135; AVX2-NEXT: retq 1136; 1137; AVX512F-LABEL: avg_v32i8_2: 1138; AVX512F: # BB#0: 1139; AVX512F-NEXT: vmovdqa (%rdi), %ymm0 1140; AVX512F-NEXT: vpavgb (%rsi), %ymm0, %ymm0 1141; AVX512F-NEXT: vmovdqu %ymm0, (%rax) 1142; AVX512F-NEXT: retq 1143; 1144; AVX512BW-LABEL: avg_v32i8_2: 1145; AVX512BW: # BB#0: 1146; AVX512BW-NEXT: vmovdqa (%rdi), %ymm0 1147; AVX512BW-NEXT: vpavgb (%rsi), %ymm0, %ymm0 1148; AVX512BW-NEXT: vmovdqu %ymm0, (%rax) 1149; AVX512BW-NEXT: retq 1150 %1 = load <32 x i8>, <32 x i8>* %a 1151 %2 = load <32 x i8>, <32 x i8>* %b 1152 %3 = zext <32 x i8> %1 to <32 x i32> 1153 %4 = zext <32 x i8> %2 to <32 x i32> 1154 %5 = add nuw nsw <32 x i32> %3, %4 1155 %6 = add nuw nsw <32 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1156 %7 = lshr <32 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1157 %8 = trunc <32 x i32> %7 to <32 x i8> 1158 store <32 x i8> %8, <32 x i8>* undef, align 4 1159 ret void 1160} 1161 1162define void @avg_v64i8_2(<64 x i8>* %a, <64 x i8>* %b) { 1163; SSE2-LABEL: avg_v64i8_2: 1164; SSE2: # BB#0: 1165; SSE2-NEXT: movdqa (%rsi), %xmm15 1166; SSE2-NEXT: movdqa 16(%rsi), %xmm13 1167; SSE2-NEXT: movdqa 32(%rsi), %xmm2 1168; SSE2-NEXT: movdqa 48(%rsi), %xmm3 1169; SSE2-NEXT: pxor %xmm0, %xmm0 1170; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm15[2,3,0,1] 1171; SSE2-NEXT: punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3],xmm15[4],xmm0[4],xmm15[5],xmm0[5],xmm15[6],xmm0[6],xmm15[7],xmm0[7] 1172; SSE2-NEXT: movdqa %xmm15, %xmm8 1173; SSE2-NEXT: punpckhwd {{.*#+}} xmm8 = xmm8[4],xmm0[4],xmm8[5],xmm0[5],xmm8[6],xmm0[6],xmm8[7],xmm0[7] 1174; SSE2-NEXT: punpcklwd {{.*#+}} xmm15 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3] 1175; SSE2-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3],xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7] 1176; SSE2-NEXT: movdqa %xmm7, %xmm14 1177; SSE2-NEXT: punpckhwd {{.*#+}} xmm14 = xmm14[4],xmm0[4],xmm14[5],xmm0[5],xmm14[6],xmm0[6],xmm14[7],xmm0[7] 1178; SSE2-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3] 1179; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm13[2,3,0,1] 1180; SSE2-NEXT: punpcklbw {{.*#+}} xmm13 = xmm13[0],xmm0[0],xmm13[1],xmm0[1],xmm13[2],xmm0[2],xmm13[3],xmm0[3],xmm13[4],xmm0[4],xmm13[5],xmm0[5],xmm13[6],xmm0[6],xmm13[7],xmm0[7] 1181; SSE2-NEXT: movdqa %xmm13, %xmm9 1182; SSE2-NEXT: punpckhwd {{.*#+}} xmm9 = xmm9[4],xmm0[4],xmm9[5],xmm0[5],xmm9[6],xmm0[6],xmm9[7],xmm0[7] 1183; SSE2-NEXT: punpcklwd {{.*#+}} xmm13 = xmm13[0],xmm0[0],xmm13[1],xmm0[1],xmm13[2],xmm0[2],xmm13[3],xmm0[3] 1184; SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3],xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7] 1185; SSE2-NEXT: movdqa %xmm6, %xmm12 1186; SSE2-NEXT: punpckhwd {{.*#+}} xmm12 = xmm12[4],xmm0[4],xmm12[5],xmm0[5],xmm12[6],xmm0[6],xmm12[7],xmm0[7] 1187; SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3] 1188; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm2[2,3,0,1] 1189; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7] 1190; SSE2-NEXT: movdqa %xmm2, %xmm11 1191; SSE2-NEXT: punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm0[4],xmm11[5],xmm0[5],xmm11[6],xmm0[6],xmm11[7],xmm0[7] 1192; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3] 1193; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7] 1194; SSE2-NEXT: movdqa %xmm5, %xmm10 1195; SSE2-NEXT: punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm0[4],xmm10[5],xmm0[5],xmm10[6],xmm0[6],xmm10[7],xmm0[7] 1196; SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3] 1197; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1] 1198; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] 1199; SSE2-NEXT: movdqa %xmm3, %xmm4 1200; SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7] 1201; SSE2-NEXT: movdqa %xmm4, -{{[0-9]+}}(%rsp) # 16-byte Spill 1202; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3] 1203; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] 1204; SSE2-NEXT: movdqa %xmm1, %xmm4 1205; SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7] 1206; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] 1207; SSE2-NEXT: paddd %xmm1, %xmm1 1208; SSE2-NEXT: paddd %xmm4, %xmm4 1209; SSE2-NEXT: movdqa %xmm4, -{{[0-9]+}}(%rsp) # 16-byte Spill 1210; SSE2-NEXT: paddd %xmm3, %xmm3 1211; SSE2-NEXT: movdqa -{{[0-9]+}}(%rsp), %xmm4 # 16-byte Reload 1212; SSE2-NEXT: paddd %xmm4, %xmm4 1213; SSE2-NEXT: paddd %xmm5, %xmm5 1214; SSE2-NEXT: paddd %xmm10, %xmm10 1215; SSE2-NEXT: paddd %xmm2, %xmm2 1216; SSE2-NEXT: paddd %xmm11, %xmm11 1217; SSE2-NEXT: paddd %xmm6, %xmm6 1218; SSE2-NEXT: paddd %xmm12, %xmm12 1219; SSE2-NEXT: paddd %xmm13, %xmm13 1220; SSE2-NEXT: paddd %xmm9, %xmm9 1221; SSE2-NEXT: paddd %xmm7, %xmm7 1222; SSE2-NEXT: paddd %xmm14, %xmm14 1223; SSE2-NEXT: paddd %xmm15, %xmm15 1224; SSE2-NEXT: paddd %xmm8, %xmm8 1225; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [1,1,1,1] 1226; SSE2-NEXT: paddd %xmm0, %xmm8 1227; SSE2-NEXT: paddd %xmm0, %xmm15 1228; SSE2-NEXT: paddd %xmm0, %xmm14 1229; SSE2-NEXT: paddd %xmm0, %xmm7 1230; SSE2-NEXT: paddd %xmm0, %xmm9 1231; SSE2-NEXT: paddd %xmm0, %xmm13 1232; SSE2-NEXT: paddd %xmm0, %xmm12 1233; SSE2-NEXT: paddd %xmm0, %xmm6 1234; SSE2-NEXT: paddd %xmm0, %xmm11 1235; SSE2-NEXT: paddd %xmm0, %xmm2 1236; SSE2-NEXT: paddd %xmm0, %xmm10 1237; SSE2-NEXT: paddd %xmm0, %xmm5 1238; SSE2-NEXT: paddd %xmm0, %xmm4 1239; SSE2-NEXT: movdqa %xmm4, -{{[0-9]+}}(%rsp) # 16-byte Spill 1240; SSE2-NEXT: paddd %xmm0, %xmm3 1241; SSE2-NEXT: movdqa -{{[0-9]+}}(%rsp), %xmm4 # 16-byte Reload 1242; SSE2-NEXT: paddd %xmm0, %xmm4 1243; SSE2-NEXT: paddd %xmm0, %xmm1 1244; SSE2-NEXT: psrld $1, %xmm15 1245; SSE2-NEXT: psrld $1, %xmm8 1246; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0] 1247; SSE2-NEXT: pand %xmm0, %xmm8 1248; SSE2-NEXT: pand %xmm0, %xmm15 1249; SSE2-NEXT: packuswb %xmm8, %xmm15 1250; SSE2-NEXT: psrld $1, %xmm7 1251; SSE2-NEXT: psrld $1, %xmm14 1252; SSE2-NEXT: pand %xmm0, %xmm14 1253; SSE2-NEXT: pand %xmm0, %xmm7 1254; SSE2-NEXT: packuswb %xmm14, %xmm7 1255; SSE2-NEXT: packuswb %xmm7, %xmm15 1256; SSE2-NEXT: psrld $1, %xmm13 1257; SSE2-NEXT: psrld $1, %xmm9 1258; SSE2-NEXT: pand %xmm0, %xmm9 1259; SSE2-NEXT: pand %xmm0, %xmm13 1260; SSE2-NEXT: packuswb %xmm9, %xmm13 1261; SSE2-NEXT: psrld $1, %xmm6 1262; SSE2-NEXT: psrld $1, %xmm12 1263; SSE2-NEXT: pand %xmm0, %xmm12 1264; SSE2-NEXT: pand %xmm0, %xmm6 1265; SSE2-NEXT: packuswb %xmm12, %xmm6 1266; SSE2-NEXT: packuswb %xmm6, %xmm13 1267; SSE2-NEXT: psrld $1, %xmm2 1268; SSE2-NEXT: psrld $1, %xmm11 1269; SSE2-NEXT: pand %xmm0, %xmm11 1270; SSE2-NEXT: pand %xmm0, %xmm2 1271; SSE2-NEXT: packuswb %xmm11, %xmm2 1272; SSE2-NEXT: psrld $1, %xmm5 1273; SSE2-NEXT: psrld $1, %xmm10 1274; SSE2-NEXT: pand %xmm0, %xmm10 1275; SSE2-NEXT: pand %xmm0, %xmm5 1276; SSE2-NEXT: packuswb %xmm10, %xmm5 1277; SSE2-NEXT: packuswb %xmm5, %xmm2 1278; SSE2-NEXT: psrld $1, %xmm3 1279; SSE2-NEXT: movdqa -{{[0-9]+}}(%rsp), %xmm5 # 16-byte Reload 1280; SSE2-NEXT: psrld $1, %xmm5 1281; SSE2-NEXT: pand %xmm0, %xmm5 1282; SSE2-NEXT: pand %xmm0, %xmm3 1283; SSE2-NEXT: packuswb %xmm5, %xmm3 1284; SSE2-NEXT: psrld $1, %xmm1 1285; SSE2-NEXT: movdqa %xmm4, %xmm5 1286; SSE2-NEXT: psrld $1, %xmm5 1287; SSE2-NEXT: pand %xmm0, %xmm5 1288; SSE2-NEXT: pand %xmm0, %xmm1 1289; SSE2-NEXT: packuswb %xmm5, %xmm1 1290; SSE2-NEXT: packuswb %xmm1, %xmm3 1291; SSE2-NEXT: movdqu %xmm3, (%rax) 1292; SSE2-NEXT: movdqu %xmm2, (%rax) 1293; SSE2-NEXT: movdqu %xmm13, (%rax) 1294; SSE2-NEXT: movdqu %xmm15, (%rax) 1295; SSE2-NEXT: retq 1296; 1297; AVX2-LABEL: avg_v64i8_2: 1298; AVX2: # BB#0: 1299; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 1300; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 1301; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 1302; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm3 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 1303; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 1304; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm5 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 1305; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm6 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 1306; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm7 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 1307; AVX2-NEXT: vpaddd %ymm7, %ymm7, %ymm7 1308; AVX2-NEXT: vpaddd %ymm6, %ymm6, %ymm6 1309; AVX2-NEXT: vpaddd %ymm5, %ymm5, %ymm5 1310; AVX2-NEXT: vpaddd %ymm4, %ymm4, %ymm4 1311; AVX2-NEXT: vpaddd %ymm3, %ymm3, %ymm3 1312; AVX2-NEXT: vpaddd %ymm2, %ymm2, %ymm2 1313; AVX2-NEXT: vpaddd %ymm1, %ymm1, %ymm1 1314; AVX2-NEXT: vpaddd %ymm0, %ymm0, %ymm0 1315; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %ymm8 1316; AVX2-NEXT: vpaddd %ymm8, %ymm0, %ymm9 1317; AVX2-NEXT: vpaddd %ymm8, %ymm1, %ymm10 1318; AVX2-NEXT: vpaddd %ymm8, %ymm2, %ymm2 1319; AVX2-NEXT: vpaddd %ymm8, %ymm3, %ymm3 1320; AVX2-NEXT: vpaddd %ymm8, %ymm4, %ymm4 1321; AVX2-NEXT: vpaddd %ymm8, %ymm5, %ymm5 1322; AVX2-NEXT: vpaddd %ymm8, %ymm6, %ymm1 1323; AVX2-NEXT: vpaddd %ymm8, %ymm7, %ymm0 1324; AVX2-NEXT: vpsrld $1, %ymm0, %ymm11 1325; AVX2-NEXT: vpsrld $1, %ymm1, %ymm12 1326; AVX2-NEXT: vpsrld $1, %ymm5, %ymm5 1327; AVX2-NEXT: vpsrld $1, %ymm4, %ymm4 1328; AVX2-NEXT: vpsrld $1, %ymm3, %ymm6 1329; AVX2-NEXT: vpsrld $1, %ymm2, %ymm7 1330; AVX2-NEXT: vpsrld $1, %ymm10, %ymm8 1331; AVX2-NEXT: vpsrld $1, %ymm9, %ymm3 1332; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [0,1,4,5,8,9,12,13,128,128,128,128,128,128,128,128,0,1,4,5,8,9,12,13,128,128,128,128,128,128,128,128] 1333; AVX2-NEXT: vpshufb %ymm2, %ymm3, %ymm3 1334; AVX2-NEXT: vpermq {{.*#+}} ymm9 = ymm3[0,2,2,3] 1335; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u> 1336; AVX2-NEXT: vpshufb %xmm3, %xmm9, %xmm0 1337; AVX2-NEXT: vpshufb %ymm2, %ymm8, %ymm8 1338; AVX2-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,2,2,3] 1339; AVX2-NEXT: vpshufb %xmm3, %xmm8, %xmm1 1340; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] 1341; AVX2-NEXT: vpshufb %ymm2, %ymm7, %ymm1 1342; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3] 1343; AVX2-NEXT: vpshufb %xmm3, %xmm1, %xmm1 1344; AVX2-NEXT: vpshufb %ymm2, %ymm6, %ymm6 1345; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm6[0,2,2,3] 1346; AVX2-NEXT: vpshufb %xmm3, %xmm6, %xmm6 1347; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm6[0],xmm1[0] 1348; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 1349; AVX2-NEXT: vpshufb %ymm2, %ymm4, %ymm1 1350; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3] 1351; AVX2-NEXT: vpshufb %xmm3, %xmm1, %xmm1 1352; AVX2-NEXT: vpshufb %ymm2, %ymm5, %ymm4 1353; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,2,3] 1354; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm4 1355; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm4[0],xmm1[0] 1356; AVX2-NEXT: vpshufb %ymm2, %ymm12, %ymm4 1357; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,2,3] 1358; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm4 1359; AVX2-NEXT: vpshufb %ymm2, %ymm11, %ymm2 1360; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3] 1361; AVX2-NEXT: vpshufb %xmm3, %xmm2, %xmm2 1362; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0] 1363; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1 1364; AVX2-NEXT: vmovdqu %ymm1, (%rax) 1365; AVX2-NEXT: vmovdqu %ymm0, (%rax) 1366; AVX2-NEXT: vzeroupper 1367; AVX2-NEXT: retq 1368; 1369; AVX512F-LABEL: avg_v64i8_2: 1370; AVX512F: # BB#0: 1371; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero 1372; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero 1373; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero 1374; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm3 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero 1375; AVX512F-NEXT: vpaddd %zmm3, %zmm3, %zmm3 1376; AVX512F-NEXT: vpaddd %zmm2, %zmm2, %zmm2 1377; AVX512F-NEXT: vpaddd %zmm1, %zmm1, %zmm1 1378; AVX512F-NEXT: vpaddd %zmm0, %zmm0, %zmm0 1379; AVX512F-NEXT: vpbroadcastd {{.*}}(%rip), %zmm4 1380; AVX512F-NEXT: vpaddd %zmm4, %zmm0, %zmm0 1381; AVX512F-NEXT: vpaddd %zmm4, %zmm1, %zmm1 1382; AVX512F-NEXT: vpaddd %zmm4, %zmm2, %zmm2 1383; AVX512F-NEXT: vpaddd %zmm4, %zmm3, %zmm3 1384; AVX512F-NEXT: vpsrld $1, %zmm3, %zmm3 1385; AVX512F-NEXT: vpsrld $1, %zmm2, %zmm2 1386; AVX512F-NEXT: vpsrld $1, %zmm1, %zmm1 1387; AVX512F-NEXT: vpsrld $1, %zmm0, %zmm0 1388; AVX512F-NEXT: vpmovdb %zmm0, %xmm0 1389; AVX512F-NEXT: vpmovdb %zmm1, %xmm1 1390; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 1391; AVX512F-NEXT: vpmovdb %zmm2, %xmm1 1392; AVX512F-NEXT: vpmovdb %zmm3, %xmm2 1393; AVX512F-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 1394; AVX512F-NEXT: vmovdqu %ymm1, (%rax) 1395; AVX512F-NEXT: vmovdqu %ymm0, (%rax) 1396; AVX512F-NEXT: retq 1397; 1398; AVX512BW-LABEL: avg_v64i8_2: 1399; AVX512BW: # BB#0: 1400; AVX512BW-NEXT: vmovdqu8 (%rsi), %zmm0 1401; AVX512BW-NEXT: vpavgb %zmm0, %zmm0, %zmm0 1402; AVX512BW-NEXT: vmovdqu8 %zmm0, (%rax) 1403; AVX512BW-NEXT: retq 1404 %1 = load <64 x i8>, <64 x i8>* %a 1405 %2 = load <64 x i8>, <64 x i8>* %b 1406 %3 = zext <64 x i8> %1 to <64 x i32> 1407 %4 = zext <64 x i8> %2 to <64 x i32> 1408 %5 = add nuw nsw <64 x i32> %4, %4 1409 %6 = add nuw nsw <64 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1410 %7 = lshr <64 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1411 %8 = trunc <64 x i32> %7 to <64 x i8> 1412 store <64 x i8> %8, <64 x i8>* undef, align 4 1413 ret void 1414} 1415 1416 1417define void @avg_v4i16_2(<4 x i16>* %a, <4 x i16>* %b) { 1418; SSE2-LABEL: avg_v4i16_2: 1419; SSE2: # BB#0: 1420; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero 1421; SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero 1422; SSE2-NEXT: pavgw %xmm0, %xmm1 1423; SSE2-NEXT: movq %xmm1, (%rax) 1424; SSE2-NEXT: retq 1425; 1426; AVX2-LABEL: avg_v4i16_2: 1427; AVX2: # BB#0: 1428; AVX2-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 1429; AVX2-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero 1430; AVX2-NEXT: vpavgw %xmm1, %xmm0, %xmm0 1431; AVX2-NEXT: vmovq %xmm0, (%rax) 1432; AVX2-NEXT: retq 1433; 1434; AVX512F-LABEL: avg_v4i16_2: 1435; AVX512F: # BB#0: 1436; AVX512F-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 1437; AVX512F-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero 1438; AVX512F-NEXT: vpavgw %xmm1, %xmm0, %xmm0 1439; AVX512F-NEXT: vmovq %xmm0, (%rax) 1440; AVX512F-NEXT: retq 1441; 1442; AVX512BW-LABEL: avg_v4i16_2: 1443; AVX512BW: # BB#0: 1444; AVX512BW-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 1445; AVX512BW-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero 1446; AVX512BW-NEXT: vpavgw %xmm1, %xmm0, %xmm0 1447; AVX512BW-NEXT: vmovq %xmm0, (%rax) 1448; AVX512BW-NEXT: retq 1449 %1 = load <4 x i16>, <4 x i16>* %a 1450 %2 = load <4 x i16>, <4 x i16>* %b 1451 %3 = zext <4 x i16> %1 to <4 x i32> 1452 %4 = zext <4 x i16> %2 to <4 x i32> 1453 %5 = add nuw nsw <4 x i32> %3, %4 1454 %6 = add nuw nsw <4 x i32> %5, <i32 1, i32 1, i32 1, i32 1> 1455 %7 = lshr <4 x i32> %6, <i32 1, i32 1, i32 1, i32 1> 1456 %8 = trunc <4 x i32> %7 to <4 x i16> 1457 store <4 x i16> %8, <4 x i16>* undef, align 4 1458 ret void 1459} 1460 1461define void @avg_v8i16_2(<8 x i16>* %a, <8 x i16>* %b) { 1462; SSE2-LABEL: avg_v8i16_2: 1463; SSE2: # BB#0: 1464; SSE2-NEXT: movdqa (%rdi), %xmm0 1465; SSE2-NEXT: pavgw (%rsi), %xmm0 1466; SSE2-NEXT: movdqu %xmm0, (%rax) 1467; SSE2-NEXT: retq 1468; 1469; AVX2-LABEL: avg_v8i16_2: 1470; AVX2: # BB#0: 1471; AVX2-NEXT: vmovdqa (%rdi), %xmm0 1472; AVX2-NEXT: vpavgw (%rsi), %xmm0, %xmm0 1473; AVX2-NEXT: vmovdqu %xmm0, (%rax) 1474; AVX2-NEXT: retq 1475; 1476; AVX512F-LABEL: avg_v8i16_2: 1477; AVX512F: # BB#0: 1478; AVX512F-NEXT: vmovdqa (%rdi), %xmm0 1479; AVX512F-NEXT: vpavgw (%rsi), %xmm0, %xmm0 1480; AVX512F-NEXT: vmovdqu %xmm0, (%rax) 1481; AVX512F-NEXT: retq 1482; 1483; AVX512BW-LABEL: avg_v8i16_2: 1484; AVX512BW: # BB#0: 1485; AVX512BW-NEXT: vmovdqa (%rdi), %xmm0 1486; AVX512BW-NEXT: vpavgw (%rsi), %xmm0, %xmm0 1487; AVX512BW-NEXT: vmovdqu %xmm0, (%rax) 1488; AVX512BW-NEXT: retq 1489 %1 = load <8 x i16>, <8 x i16>* %a 1490 %2 = load <8 x i16>, <8 x i16>* %b 1491 %3 = zext <8 x i16> %1 to <8 x i32> 1492 %4 = zext <8 x i16> %2 to <8 x i32> 1493 %5 = add nuw nsw <8 x i32> %3, %4 1494 %6 = add nuw nsw <8 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1495 %7 = lshr <8 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1496 %8 = trunc <8 x i32> %7 to <8 x i16> 1497 store <8 x i16> %8, <8 x i16>* undef, align 4 1498 ret void 1499} 1500 1501define void @avg_v16i16_2(<16 x i16>* %a, <16 x i16>* %b) { 1502; SSE2-LABEL: avg_v16i16_2: 1503; SSE2: # BB#0: 1504; SSE2-NEXT: movdqa (%rdi), %xmm4 1505; SSE2-NEXT: movdqa 16(%rdi), %xmm5 1506; SSE2-NEXT: movdqa (%rsi), %xmm0 1507; SSE2-NEXT: movdqa 16(%rsi), %xmm1 1508; SSE2-NEXT: pxor %xmm6, %xmm6 1509; SSE2-NEXT: movdqa %xmm4, %xmm8 1510; SSE2-NEXT: punpckhwd {{.*#+}} xmm8 = xmm8[4],xmm6[4],xmm8[5],xmm6[5],xmm8[6],xmm6[6],xmm8[7],xmm6[7] 1511; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3] 1512; SSE2-NEXT: movdqa %xmm5, %xmm7 1513; SSE2-NEXT: punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7] 1514; SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3] 1515; SSE2-NEXT: movdqa %xmm0, %xmm3 1516; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm6[4],xmm3[5],xmm6[5],xmm3[6],xmm6[6],xmm3[7],xmm6[7] 1517; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3] 1518; SSE2-NEXT: movdqa %xmm1, %xmm2 1519; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7] 1520; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1],xmm1[2],xmm6[2],xmm1[3],xmm6[3] 1521; SSE2-NEXT: paddd %xmm5, %xmm1 1522; SSE2-NEXT: paddd %xmm7, %xmm2 1523; SSE2-NEXT: paddd %xmm4, %xmm0 1524; SSE2-NEXT: paddd %xmm8, %xmm3 1525; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [1,1,1,1] 1526; SSE2-NEXT: paddd %xmm4, %xmm3 1527; SSE2-NEXT: paddd %xmm4, %xmm0 1528; SSE2-NEXT: paddd %xmm4, %xmm2 1529; SSE2-NEXT: paddd %xmm4, %xmm1 1530; SSE2-NEXT: psrld $1, %xmm1 1531; SSE2-NEXT: psrld $1, %xmm2 1532; SSE2-NEXT: psrld $1, %xmm0 1533; SSE2-NEXT: psrld $1, %xmm3 1534; SSE2-NEXT: pslld $16, %xmm3 1535; SSE2-NEXT: psrad $16, %xmm3 1536; SSE2-NEXT: pslld $16, %xmm0 1537; SSE2-NEXT: psrad $16, %xmm0 1538; SSE2-NEXT: packssdw %xmm3, %xmm0 1539; SSE2-NEXT: pslld $16, %xmm2 1540; SSE2-NEXT: psrad $16, %xmm2 1541; SSE2-NEXT: pslld $16, %xmm1 1542; SSE2-NEXT: psrad $16, %xmm1 1543; SSE2-NEXT: packssdw %xmm2, %xmm1 1544; SSE2-NEXT: movdqu %xmm1, (%rax) 1545; SSE2-NEXT: movdqu %xmm0, (%rax) 1546; SSE2-NEXT: retq 1547; 1548; AVX2-LABEL: avg_v16i16_2: 1549; AVX2: # BB#0: 1550; AVX2-NEXT: vmovdqa (%rdi), %ymm0 1551; AVX2-NEXT: vpavgw (%rsi), %ymm0, %ymm0 1552; AVX2-NEXT: vmovdqu %ymm0, (%rax) 1553; AVX2-NEXT: vzeroupper 1554; AVX2-NEXT: retq 1555; 1556; AVX512F-LABEL: avg_v16i16_2: 1557; AVX512F: # BB#0: 1558; AVX512F-NEXT: vmovdqa (%rdi), %ymm0 1559; AVX512F-NEXT: vpavgw (%rsi), %ymm0, %ymm0 1560; AVX512F-NEXT: vmovdqu %ymm0, (%rax) 1561; AVX512F-NEXT: retq 1562; 1563; AVX512BW-LABEL: avg_v16i16_2: 1564; AVX512BW: # BB#0: 1565; AVX512BW-NEXT: vmovdqa (%rdi), %ymm0 1566; AVX512BW-NEXT: vpavgw (%rsi), %ymm0, %ymm0 1567; AVX512BW-NEXT: vmovdqu %ymm0, (%rax) 1568; AVX512BW-NEXT: retq 1569 %1 = load <16 x i16>, <16 x i16>* %a 1570 %2 = load <16 x i16>, <16 x i16>* %b 1571 %3 = zext <16 x i16> %1 to <16 x i32> 1572 %4 = zext <16 x i16> %2 to <16 x i32> 1573 %5 = add nuw nsw <16 x i32> %3, %4 1574 %6 = add nuw nsw <16 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1575 %7 = lshr <16 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1576 %8 = trunc <16 x i32> %7 to <16 x i16> 1577 store <16 x i16> %8, <16 x i16>* undef, align 4 1578 ret void 1579} 1580 1581define void @avg_v32i16_2(<32 x i16>* %a, <32 x i16>* %b) { 1582; SSE2-LABEL: avg_v32i16_2: 1583; SSE2: # BB#0: 1584; SSE2-NEXT: movdqa (%rdi), %xmm10 1585; SSE2-NEXT: movdqa 16(%rdi), %xmm9 1586; SSE2-NEXT: movdqa 32(%rdi), %xmm11 1587; SSE2-NEXT: movdqa 48(%rdi), %xmm8 1588; SSE2-NEXT: movdqa (%rsi), %xmm14 1589; SSE2-NEXT: movdqa 16(%rsi), %xmm1 1590; SSE2-NEXT: movdqa 32(%rsi), %xmm2 1591; SSE2-NEXT: movdqa 48(%rsi), %xmm3 1592; SSE2-NEXT: pxor %xmm0, %xmm0 1593; SSE2-NEXT: movdqa %xmm10, %xmm4 1594; SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7] 1595; SSE2-NEXT: movdqa %xmm4, -{{[0-9]+}}(%rsp) # 16-byte Spill 1596; SSE2-NEXT: punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm0[0],xmm10[1],xmm0[1],xmm10[2],xmm0[2],xmm10[3],xmm0[3] 1597; SSE2-NEXT: movdqa %xmm9, %xmm12 1598; SSE2-NEXT: punpckhwd {{.*#+}} xmm12 = xmm12[4],xmm0[4],xmm12[5],xmm0[5],xmm12[6],xmm0[6],xmm12[7],xmm0[7] 1599; SSE2-NEXT: punpcklwd {{.*#+}} xmm9 = xmm9[0],xmm0[0],xmm9[1],xmm0[1],xmm9[2],xmm0[2],xmm9[3],xmm0[3] 1600; SSE2-NEXT: movdqa %xmm11, %xmm15 1601; SSE2-NEXT: punpckhwd {{.*#+}} xmm15 = xmm15[4],xmm0[4],xmm15[5],xmm0[5],xmm15[6],xmm0[6],xmm15[7],xmm0[7] 1602; SSE2-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm0[0],xmm11[1],xmm0[1],xmm11[2],xmm0[2],xmm11[3],xmm0[3] 1603; SSE2-NEXT: movdqa %xmm8, %xmm13 1604; SSE2-NEXT: punpckhwd {{.*#+}} xmm13 = xmm13[4],xmm0[4],xmm13[5],xmm0[5],xmm13[6],xmm0[6],xmm13[7],xmm0[7] 1605; SSE2-NEXT: punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3] 1606; SSE2-NEXT: movdqa %xmm14, %xmm7 1607; SSE2-NEXT: punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7] 1608; SSE2-NEXT: punpcklwd {{.*#+}} xmm14 = xmm14[0],xmm0[0],xmm14[1],xmm0[1],xmm14[2],xmm0[2],xmm14[3],xmm0[3] 1609; SSE2-NEXT: movdqa %xmm1, %xmm6 1610; SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7] 1611; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] 1612; SSE2-NEXT: movdqa %xmm2, %xmm5 1613; SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7] 1614; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3] 1615; SSE2-NEXT: movdqa %xmm3, %xmm4 1616; SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7] 1617; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3] 1618; SSE2-NEXT: paddd %xmm8, %xmm3 1619; SSE2-NEXT: paddd %xmm13, %xmm4 1620; SSE2-NEXT: paddd %xmm11, %xmm2 1621; SSE2-NEXT: paddd %xmm15, %xmm5 1622; SSE2-NEXT: paddd %xmm9, %xmm1 1623; SSE2-NEXT: paddd %xmm12, %xmm6 1624; SSE2-NEXT: paddd %xmm10, %xmm14 1625; SSE2-NEXT: paddd -{{[0-9]+}}(%rsp), %xmm7 # 16-byte Folded Reload 1626; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [1,1,1,1] 1627; SSE2-NEXT: paddd %xmm0, %xmm7 1628; SSE2-NEXT: paddd %xmm0, %xmm14 1629; SSE2-NEXT: paddd %xmm0, %xmm6 1630; SSE2-NEXT: paddd %xmm0, %xmm1 1631; SSE2-NEXT: paddd %xmm0, %xmm5 1632; SSE2-NEXT: paddd %xmm0, %xmm2 1633; SSE2-NEXT: paddd %xmm0, %xmm4 1634; SSE2-NEXT: paddd %xmm0, %xmm3 1635; SSE2-NEXT: psrld $1, %xmm14 1636; SSE2-NEXT: psrld $1, %xmm7 1637; SSE2-NEXT: pslld $16, %xmm7 1638; SSE2-NEXT: psrad $16, %xmm7 1639; SSE2-NEXT: pslld $16, %xmm14 1640; SSE2-NEXT: psrad $16, %xmm14 1641; SSE2-NEXT: packssdw %xmm7, %xmm14 1642; SSE2-NEXT: psrld $1, %xmm1 1643; SSE2-NEXT: psrld $1, %xmm6 1644; SSE2-NEXT: pslld $16, %xmm6 1645; SSE2-NEXT: psrad $16, %xmm6 1646; SSE2-NEXT: pslld $16, %xmm1 1647; SSE2-NEXT: psrad $16, %xmm1 1648; SSE2-NEXT: packssdw %xmm6, %xmm1 1649; SSE2-NEXT: psrld $1, %xmm2 1650; SSE2-NEXT: psrld $1, %xmm5 1651; SSE2-NEXT: pslld $16, %xmm5 1652; SSE2-NEXT: psrad $16, %xmm5 1653; SSE2-NEXT: pslld $16, %xmm2 1654; SSE2-NEXT: psrad $16, %xmm2 1655; SSE2-NEXT: packssdw %xmm5, %xmm2 1656; SSE2-NEXT: psrld $1, %xmm3 1657; SSE2-NEXT: psrld $1, %xmm4 1658; SSE2-NEXT: pslld $16, %xmm4 1659; SSE2-NEXT: psrad $16, %xmm4 1660; SSE2-NEXT: pslld $16, %xmm3 1661; SSE2-NEXT: psrad $16, %xmm3 1662; SSE2-NEXT: packssdw %xmm4, %xmm3 1663; SSE2-NEXT: movdqu %xmm3, (%rax) 1664; SSE2-NEXT: movdqu %xmm2, (%rax) 1665; SSE2-NEXT: movdqu %xmm1, (%rax) 1666; SSE2-NEXT: movdqu %xmm14, (%rax) 1667; SSE2-NEXT: retq 1668; 1669; AVX2-LABEL: avg_v32i16_2: 1670; AVX2: # BB#0: 1671; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 1672; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 1673; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 1674; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 1675; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 1676; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm5 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 1677; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm6 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 1678; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm7 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 1679; AVX2-NEXT: vpaddd %ymm7, %ymm3, %ymm3 1680; AVX2-NEXT: vpaddd %ymm6, %ymm2, %ymm2 1681; AVX2-NEXT: vpaddd %ymm5, %ymm1, %ymm1 1682; AVX2-NEXT: vpaddd %ymm4, %ymm0, %ymm0 1683; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %ymm4 1684; AVX2-NEXT: vpaddd %ymm4, %ymm0, %ymm0 1685; AVX2-NEXT: vpaddd %ymm4, %ymm1, %ymm1 1686; AVX2-NEXT: vpaddd %ymm4, %ymm2, %ymm2 1687; AVX2-NEXT: vpaddd %ymm4, %ymm3, %ymm3 1688; AVX2-NEXT: vpsrld $1, %ymm3, %ymm3 1689; AVX2-NEXT: vpsrld $1, %ymm2, %ymm2 1690; AVX2-NEXT: vpsrld $1, %ymm1, %ymm1 1691; AVX2-NEXT: vpsrld $1, %ymm0, %ymm0 1692; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,4,5,8,9,12,13,128,128,128,128,128,128,128,128,0,1,4,5,8,9,12,13,128,128,128,128,128,128,128,128] 1693; AVX2-NEXT: vpshufb %ymm4, %ymm0, %ymm0 1694; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3] 1695; AVX2-NEXT: vpshufb %ymm4, %ymm1, %ymm1 1696; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3] 1697; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 1698; AVX2-NEXT: vpshufb %ymm4, %ymm2, %ymm1 1699; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3] 1700; AVX2-NEXT: vpshufb %ymm4, %ymm3, %ymm2 1701; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3] 1702; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 1703; AVX2-NEXT: vmovdqu %ymm1, (%rax) 1704; AVX2-NEXT: vmovdqu %ymm0, (%rax) 1705; AVX2-NEXT: vzeroupper 1706; AVX2-NEXT: retq 1707; 1708; AVX512F-LABEL: avg_v32i16_2: 1709; AVX512F: # BB#0: 1710; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero 1711; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero 1712; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero 1713; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero 1714; AVX512F-NEXT: vpaddd %zmm3, %zmm1, %zmm1 1715; AVX512F-NEXT: vpaddd %zmm2, %zmm0, %zmm0 1716; AVX512F-NEXT: vpbroadcastd {{.*}}(%rip), %zmm2 1717; AVX512F-NEXT: vpaddd %zmm2, %zmm0, %zmm0 1718; AVX512F-NEXT: vpaddd %zmm2, %zmm1, %zmm1 1719; AVX512F-NEXT: vpsrld $1, %zmm1, %zmm1 1720; AVX512F-NEXT: vpsrld $1, %zmm0, %zmm0 1721; AVX512F-NEXT: vpmovdw %zmm0, (%rax) 1722; AVX512F-NEXT: vpmovdw %zmm1, (%rax) 1723; AVX512F-NEXT: retq 1724; 1725; AVX512BW-LABEL: avg_v32i16_2: 1726; AVX512BW: # BB#0: 1727; AVX512BW-NEXT: vmovdqu16 (%rdi), %zmm0 1728; AVX512BW-NEXT: vpavgw (%rsi), %zmm0, %zmm0 1729; AVX512BW-NEXT: vmovdqu16 %zmm0, (%rax) 1730; AVX512BW-NEXT: retq 1731 %1 = load <32 x i16>, <32 x i16>* %a 1732 %2 = load <32 x i16>, <32 x i16>* %b 1733 %3 = zext <32 x i16> %1 to <32 x i32> 1734 %4 = zext <32 x i16> %2 to <32 x i32> 1735 %5 = add nuw nsw <32 x i32> %3, %4 1736 %6 = add nuw nsw <32 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1737 %7 = lshr <32 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1738 %8 = trunc <32 x i32> %7 to <32 x i16> 1739 store <32 x i16> %8, <32 x i16>* undef, align 4 1740 ret void 1741} 1742 1743define void @avg_v4i8_const(<4 x i8>* %a) { 1744; SSE2-LABEL: avg_v4i8_const: 1745; SSE2: # BB#0: 1746; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero 1747; SSE2-NEXT: pavgb {{.*}}(%rip), %xmm0 1748; SSE2-NEXT: movd %xmm0, (%rax) 1749; SSE2-NEXT: retq 1750; 1751; AVX2-LABEL: avg_v4i8_const: 1752; AVX2: # BB#0: 1753; AVX2-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero 1754; AVX2-NEXT: vpavgb {{.*}}(%rip), %xmm0, %xmm0 1755; AVX2-NEXT: vmovd %xmm0, (%rax) 1756; AVX2-NEXT: retq 1757; 1758; AVX512F-LABEL: avg_v4i8_const: 1759; AVX512F: # BB#0: 1760; AVX512F-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero 1761; AVX512F-NEXT: vpavgb {{.*}}(%rip), %xmm0, %xmm0 1762; AVX512F-NEXT: vmovd %xmm0, (%rax) 1763; AVX512F-NEXT: retq 1764; 1765; AVX512BW-LABEL: avg_v4i8_const: 1766; AVX512BW: # BB#0: 1767; AVX512BW-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero 1768; AVX512BW-NEXT: vpavgb {{.*}}(%rip), %xmm0, %xmm0 1769; AVX512BW-NEXT: vmovd %xmm0, (%rax) 1770; AVX512BW-NEXT: retq 1771 %1 = load <4 x i8>, <4 x i8>* %a 1772 %2 = zext <4 x i8> %1 to <4 x i32> 1773 %3 = add nuw nsw <4 x i32> %2, <i32 1, i32 2, i32 3, i32 4> 1774 %4 = lshr <4 x i32> %3, <i32 1, i32 1, i32 1, i32 1> 1775 %5 = trunc <4 x i32> %4 to <4 x i8> 1776 store <4 x i8> %5, <4 x i8>* undef, align 4 1777 ret void 1778} 1779 1780define void @avg_v8i8_const(<8 x i8>* %a) { 1781; SSE2-LABEL: avg_v8i8_const: 1782; SSE2: # BB#0: 1783; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero 1784; SSE2-NEXT: pavgb {{.*}}(%rip), %xmm0 1785; SSE2-NEXT: movq %xmm0, (%rax) 1786; SSE2-NEXT: retq 1787; 1788; AVX2-LABEL: avg_v8i8_const: 1789; AVX2: # BB#0: 1790; AVX2-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 1791; AVX2-NEXT: vpavgb {{.*}}(%rip), %xmm0, %xmm0 1792; AVX2-NEXT: vmovq %xmm0, (%rax) 1793; AVX2-NEXT: retq 1794; 1795; AVX512F-LABEL: avg_v8i8_const: 1796; AVX512F: # BB#0: 1797; AVX512F-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 1798; AVX512F-NEXT: vpavgb {{.*}}(%rip), %xmm0, %xmm0 1799; AVX512F-NEXT: vmovq %xmm0, (%rax) 1800; AVX512F-NEXT: retq 1801; 1802; AVX512BW-LABEL: avg_v8i8_const: 1803; AVX512BW: # BB#0: 1804; AVX512BW-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 1805; AVX512BW-NEXT: vpavgb {{.*}}(%rip), %xmm0, %xmm0 1806; AVX512BW-NEXT: vmovq %xmm0, (%rax) 1807; AVX512BW-NEXT: retq 1808 %1 = load <8 x i8>, <8 x i8>* %a 1809 %2 = zext <8 x i8> %1 to <8 x i32> 1810 %3 = add nuw nsw <8 x i32> %2, <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8> 1811 %4 = lshr <8 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1812 %5 = trunc <8 x i32> %4 to <8 x i8> 1813 store <8 x i8> %5, <8 x i8>* undef, align 4 1814 ret void 1815} 1816 1817define void @avg_v16i8_const(<16 x i8>* %a) { 1818; SSE2-LABEL: avg_v16i8_const: 1819; SSE2: # BB#0: 1820; SSE2-NEXT: movdqa (%rdi), %xmm0 1821; SSE2-NEXT: pavgb {{.*}}(%rip), %xmm0 1822; SSE2-NEXT: movdqu %xmm0, (%rax) 1823; SSE2-NEXT: retq 1824; 1825; AVX2-LABEL: avg_v16i8_const: 1826; AVX2: # BB#0: 1827; AVX2-NEXT: vmovdqa (%rdi), %xmm0 1828; AVX2-NEXT: vpavgb {{.*}}(%rip), %xmm0, %xmm0 1829; AVX2-NEXT: vmovdqu %xmm0, (%rax) 1830; AVX2-NEXT: retq 1831; 1832; AVX512F-LABEL: avg_v16i8_const: 1833; AVX512F: # BB#0: 1834; AVX512F-NEXT: vmovdqa (%rdi), %xmm0 1835; AVX512F-NEXT: vpavgb {{.*}}(%rip), %xmm0, %xmm0 1836; AVX512F-NEXT: vmovdqu %xmm0, (%rax) 1837; AVX512F-NEXT: retq 1838; 1839; AVX512BW-LABEL: avg_v16i8_const: 1840; AVX512BW: # BB#0: 1841; AVX512BW-NEXT: vmovdqa (%rdi), %xmm0 1842; AVX512BW-NEXT: vpavgb {{.*}}(%rip), %xmm0, %xmm0 1843; AVX512BW-NEXT: vmovdqu %xmm0, (%rax) 1844; AVX512BW-NEXT: retq 1845 %1 = load <16 x i8>, <16 x i8>* %a 1846 %2 = zext <16 x i8> %1 to <16 x i32> 1847 %3 = add nuw nsw <16 x i32> %2, <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8> 1848 %4 = lshr <16 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1849 %5 = trunc <16 x i32> %4 to <16 x i8> 1850 store <16 x i8> %5, <16 x i8>* undef, align 4 1851 ret void 1852} 1853 1854define void @avg_v32i8_const(<32 x i8>* %a) { 1855; SSE2-LABEL: avg_v32i8_const: 1856; SSE2: # BB#0: 1857; SSE2-NEXT: movdqa (%rdi), %xmm4 1858; SSE2-NEXT: movdqa 16(%rdi), %xmm2 1859; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,0,1] 1860; SSE2-NEXT: pxor %xmm1, %xmm1 1861; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] 1862; SSE2-NEXT: movdqa %xmm0, %xmm8 1863; SSE2-NEXT: punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm1[0],xmm8[1],xmm1[1],xmm8[2],xmm1[2],xmm8[3],xmm1[3] 1864; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] 1865; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] 1866; SSE2-NEXT: movdqa %xmm2, %xmm3 1867; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3] 1868; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] 1869; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm4[2,3,0,1] 1870; SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1],xmm6[2],xmm1[2],xmm6[3],xmm1[3],xmm6[4],xmm1[4],xmm6[5],xmm1[5],xmm6[6],xmm1[6],xmm6[7],xmm1[7] 1871; SSE2-NEXT: movdqa %xmm6, %xmm7 1872; SSE2-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm1[0],xmm7[1],xmm1[1],xmm7[2],xmm1[2],xmm7[3],xmm1[3] 1873; SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm1[4],xmm6[5],xmm1[5],xmm6[6],xmm1[6],xmm6[7],xmm1[7] 1874; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3],xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7] 1875; SSE2-NEXT: movdqa %xmm4, %xmm5 1876; SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3] 1877; SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7] 1878; SSE2-NEXT: movdqa {{.*#+}} xmm9 = [5,6,7,8] 1879; SSE2-NEXT: paddd %xmm9, %xmm4 1880; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [1,2,3,4] 1881; SSE2-NEXT: paddd %xmm1, %xmm5 1882; SSE2-NEXT: paddd %xmm9, %xmm6 1883; SSE2-NEXT: paddd %xmm1, %xmm7 1884; SSE2-NEXT: paddd %xmm9, %xmm2 1885; SSE2-NEXT: paddd %xmm1, %xmm3 1886; SSE2-NEXT: paddd %xmm9, %xmm0 1887; SSE2-NEXT: paddd %xmm1, %xmm8 1888; SSE2-NEXT: psrld $1, %xmm8 1889; SSE2-NEXT: psrld $1, %xmm0 1890; SSE2-NEXT: psrld $1, %xmm3 1891; SSE2-NEXT: psrld $1, %xmm2 1892; SSE2-NEXT: psrld $1, %xmm7 1893; SSE2-NEXT: psrld $1, %xmm6 1894; SSE2-NEXT: psrld $1, %xmm5 1895; SSE2-NEXT: psrld $1, %xmm4 1896; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0] 1897; SSE2-NEXT: pand %xmm1, %xmm4 1898; SSE2-NEXT: pand %xmm1, %xmm5 1899; SSE2-NEXT: packuswb %xmm4, %xmm5 1900; SSE2-NEXT: pand %xmm1, %xmm6 1901; SSE2-NEXT: pand %xmm1, %xmm7 1902; SSE2-NEXT: packuswb %xmm6, %xmm7 1903; SSE2-NEXT: packuswb %xmm7, %xmm5 1904; SSE2-NEXT: pand %xmm1, %xmm2 1905; SSE2-NEXT: pand %xmm1, %xmm3 1906; SSE2-NEXT: packuswb %xmm2, %xmm3 1907; SSE2-NEXT: pand %xmm1, %xmm0 1908; SSE2-NEXT: pand %xmm1, %xmm8 1909; SSE2-NEXT: packuswb %xmm0, %xmm8 1910; SSE2-NEXT: packuswb %xmm8, %xmm3 1911; SSE2-NEXT: movdqu %xmm3, (%rax) 1912; SSE2-NEXT: movdqu %xmm5, (%rax) 1913; SSE2-NEXT: retq 1914; 1915; AVX2-LABEL: avg_v32i8_const: 1916; AVX2: # BB#0: 1917; AVX2-NEXT: vmovdqa (%rdi), %ymm0 1918; AVX2-NEXT: vpavgb {{.*}}(%rip), %ymm0, %ymm0 1919; AVX2-NEXT: vmovdqu %ymm0, (%rax) 1920; AVX2-NEXT: vzeroupper 1921; AVX2-NEXT: retq 1922; 1923; AVX512F-LABEL: avg_v32i8_const: 1924; AVX512F: # BB#0: 1925; AVX512F-NEXT: vmovdqa (%rdi), %ymm0 1926; AVX512F-NEXT: vpavgb {{.*}}(%rip), %ymm0, %ymm0 1927; AVX512F-NEXT: vmovdqu %ymm0, (%rax) 1928; AVX512F-NEXT: retq 1929; 1930; AVX512BW-LABEL: avg_v32i8_const: 1931; AVX512BW: # BB#0: 1932; AVX512BW-NEXT: vmovdqa (%rdi), %ymm0 1933; AVX512BW-NEXT: vpavgb {{.*}}(%rip), %ymm0, %ymm0 1934; AVX512BW-NEXT: vmovdqu %ymm0, (%rax) 1935; AVX512BW-NEXT: retq 1936 %1 = load <32 x i8>, <32 x i8>* %a 1937 %2 = zext <32 x i8> %1 to <32 x i32> 1938 %3 = add nuw nsw <32 x i32> %2, <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8> 1939 %4 = lshr <32 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1940 %5 = trunc <32 x i32> %4 to <32 x i8> 1941 store <32 x i8> %5, <32 x i8>* undef, align 4 1942 ret void 1943} 1944 1945define void @avg_v64i8_const(<64 x i8>* %a) { 1946; SSE2-LABEL: avg_v64i8_const: 1947; SSE2: # BB#0: 1948; SSE2-NEXT: movdqa (%rdi), %xmm7 1949; SSE2-NEXT: movdqa 16(%rdi), %xmm1 1950; SSE2-NEXT: movdqa 32(%rdi), %xmm14 1951; SSE2-NEXT: movdqa 48(%rdi), %xmm11 1952; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm11[2,3,0,1] 1953; SSE2-NEXT: pxor %xmm0, %xmm0 1954; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] 1955; SSE2-NEXT: movdqa %xmm3, %xmm2 1956; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3] 1957; SSE2-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp) # 16-byte Spill 1958; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] 1959; SSE2-NEXT: movdqa %xmm3, %xmm9 1960; SSE2-NEXT: punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm0[0],xmm11[1],xmm0[1],xmm11[2],xmm0[2],xmm11[3],xmm0[3],xmm11[4],xmm0[4],xmm11[5],xmm0[5],xmm11[6],xmm0[6],xmm11[7],xmm0[7] 1961; SSE2-NEXT: movdqa %xmm11, %xmm12 1962; SSE2-NEXT: punpcklwd {{.*#+}} xmm12 = xmm12[0],xmm0[0],xmm12[1],xmm0[1],xmm12[2],xmm0[2],xmm12[3],xmm0[3] 1963; SSE2-NEXT: punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm0[4],xmm11[5],xmm0[5],xmm11[6],xmm0[6],xmm11[7],xmm0[7] 1964; SSE2-NEXT: pshufd {{.*#+}} xmm13 = xmm14[2,3,0,1] 1965; SSE2-NEXT: punpcklbw {{.*#+}} xmm13 = xmm13[0],xmm0[0],xmm13[1],xmm0[1],xmm13[2],xmm0[2],xmm13[3],xmm0[3],xmm13[4],xmm0[4],xmm13[5],xmm0[5],xmm13[6],xmm0[6],xmm13[7],xmm0[7] 1966; SSE2-NEXT: movdqa %xmm13, %xmm10 1967; SSE2-NEXT: punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm0[0],xmm10[1],xmm0[1],xmm10[2],xmm0[2],xmm10[3],xmm0[3] 1968; SSE2-NEXT: punpckhwd {{.*#+}} xmm13 = xmm13[4],xmm0[4],xmm13[5],xmm0[5],xmm13[6],xmm0[6],xmm13[7],xmm0[7] 1969; SSE2-NEXT: punpcklbw {{.*#+}} xmm14 = xmm14[0],xmm0[0],xmm14[1],xmm0[1],xmm14[2],xmm0[2],xmm14[3],xmm0[3],xmm14[4],xmm0[4],xmm14[5],xmm0[5],xmm14[6],xmm0[6],xmm14[7],xmm0[7] 1970; SSE2-NEXT: movdqa %xmm14, %xmm15 1971; SSE2-NEXT: punpcklwd {{.*#+}} xmm15 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3] 1972; SSE2-NEXT: punpckhwd {{.*#+}} xmm14 = xmm14[4],xmm0[4],xmm14[5],xmm0[5],xmm14[6],xmm0[6],xmm14[7],xmm0[7] 1973; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1] 1974; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] 1975; SSE2-NEXT: movdqa %xmm3, %xmm6 1976; SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3] 1977; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7] 1978; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] 1979; SSE2-NEXT: movdqa %xmm1, %xmm8 1980; SSE2-NEXT: punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3] 1981; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] 1982; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm7[2,3,0,1] 1983; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7] 1984; SSE2-NEXT: movdqa %xmm4, %xmm5 1985; SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3] 1986; SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7] 1987; SSE2-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3],xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7] 1988; SSE2-NEXT: movdqa %xmm7, %xmm2 1989; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3] 1990; SSE2-NEXT: punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7] 1991; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [5,6,7,8] 1992; SSE2-NEXT: paddd %xmm0, %xmm7 1993; SSE2-NEXT: paddd %xmm0, %xmm4 1994; SSE2-NEXT: paddd %xmm0, %xmm1 1995; SSE2-NEXT: paddd %xmm0, %xmm3 1996; SSE2-NEXT: paddd %xmm0, %xmm14 1997; SSE2-NEXT: paddd %xmm0, %xmm13 1998; SSE2-NEXT: paddd %xmm0, %xmm11 1999; SSE2-NEXT: paddd %xmm0, %xmm9 2000; SSE2-NEXT: movdqa %xmm9, -{{[0-9]+}}(%rsp) # 16-byte Spill 2001; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [1,2,3,4] 2002; SSE2-NEXT: paddd %xmm0, %xmm2 2003; SSE2-NEXT: paddd %xmm0, %xmm5 2004; SSE2-NEXT: paddd %xmm0, %xmm8 2005; SSE2-NEXT: paddd %xmm0, %xmm6 2006; SSE2-NEXT: paddd %xmm0, %xmm15 2007; SSE2-NEXT: paddd %xmm0, %xmm10 2008; SSE2-NEXT: paddd %xmm0, %xmm12 2009; SSE2-NEXT: movdqa -{{[0-9]+}}(%rsp), %xmm9 # 16-byte Reload 2010; SSE2-NEXT: paddd %xmm0, %xmm9 2011; SSE2-NEXT: movdqa %xmm9, -{{[0-9]+}}(%rsp) # 16-byte Spill 2012; SSE2-NEXT: psrld $1, %xmm2 2013; SSE2-NEXT: psrld $1, %xmm7 2014; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0] 2015; SSE2-NEXT: pand %xmm0, %xmm7 2016; SSE2-NEXT: pand %xmm0, %xmm2 2017; SSE2-NEXT: packuswb %xmm7, %xmm2 2018; SSE2-NEXT: psrld $1, %xmm5 2019; SSE2-NEXT: psrld $1, %xmm4 2020; SSE2-NEXT: pand %xmm0, %xmm4 2021; SSE2-NEXT: pand %xmm0, %xmm5 2022; SSE2-NEXT: packuswb %xmm4, %xmm5 2023; SSE2-NEXT: packuswb %xmm5, %xmm2 2024; SSE2-NEXT: psrld $1, %xmm8 2025; SSE2-NEXT: psrld $1, %xmm1 2026; SSE2-NEXT: pand %xmm0, %xmm1 2027; SSE2-NEXT: pand %xmm0, %xmm8 2028; SSE2-NEXT: packuswb %xmm1, %xmm8 2029; SSE2-NEXT: psrld $1, %xmm6 2030; SSE2-NEXT: psrld $1, %xmm3 2031; SSE2-NEXT: pand %xmm0, %xmm3 2032; SSE2-NEXT: pand %xmm0, %xmm6 2033; SSE2-NEXT: packuswb %xmm3, %xmm6 2034; SSE2-NEXT: packuswb %xmm6, %xmm8 2035; SSE2-NEXT: psrld $1, %xmm15 2036; SSE2-NEXT: psrld $1, %xmm14 2037; SSE2-NEXT: pand %xmm0, %xmm14 2038; SSE2-NEXT: pand %xmm0, %xmm15 2039; SSE2-NEXT: packuswb %xmm14, %xmm15 2040; SSE2-NEXT: psrld $1, %xmm10 2041; SSE2-NEXT: psrld $1, %xmm13 2042; SSE2-NEXT: pand %xmm0, %xmm13 2043; SSE2-NEXT: pand %xmm0, %xmm10 2044; SSE2-NEXT: packuswb %xmm13, %xmm10 2045; SSE2-NEXT: packuswb %xmm10, %xmm15 2046; SSE2-NEXT: psrld $1, %xmm12 2047; SSE2-NEXT: psrld $1, %xmm11 2048; SSE2-NEXT: pand %xmm0, %xmm11 2049; SSE2-NEXT: pand %xmm0, %xmm12 2050; SSE2-NEXT: packuswb %xmm11, %xmm12 2051; SSE2-NEXT: movdqa -{{[0-9]+}}(%rsp), %xmm1 # 16-byte Reload 2052; SSE2-NEXT: psrld $1, %xmm1 2053; SSE2-NEXT: movdqa -{{[0-9]+}}(%rsp), %xmm3 # 16-byte Reload 2054; SSE2-NEXT: psrld $1, %xmm3 2055; SSE2-NEXT: pand %xmm0, %xmm3 2056; SSE2-NEXT: pand %xmm0, %xmm1 2057; SSE2-NEXT: packuswb %xmm3, %xmm1 2058; SSE2-NEXT: packuswb %xmm1, %xmm12 2059; SSE2-NEXT: movdqu %xmm12, (%rax) 2060; SSE2-NEXT: movdqu %xmm15, (%rax) 2061; SSE2-NEXT: movdqu %xmm8, (%rax) 2062; SSE2-NEXT: movdqu %xmm2, (%rax) 2063; SSE2-NEXT: retq 2064; 2065; AVX2-LABEL: avg_v64i8_const: 2066; AVX2: # BB#0: 2067; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 2068; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 2069; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 2070; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm3 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 2071; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 2072; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm5 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 2073; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm6 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 2074; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm7 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero 2075; AVX2-NEXT: vmovdqa {{.*#+}} ymm8 = [1,2,3,4,5,6,7,8] 2076; AVX2-NEXT: vpaddd %ymm8, %ymm7, %ymm7 2077; AVX2-NEXT: vpaddd %ymm8, %ymm6, %ymm6 2078; AVX2-NEXT: vpaddd %ymm8, %ymm5, %ymm5 2079; AVX2-NEXT: vpaddd %ymm8, %ymm4, %ymm4 2080; AVX2-NEXT: vpaddd %ymm8, %ymm3, %ymm3 2081; AVX2-NEXT: vpaddd %ymm8, %ymm2, %ymm2 2082; AVX2-NEXT: vpaddd %ymm8, %ymm1, %ymm1 2083; AVX2-NEXT: vpaddd %ymm8, %ymm0, %ymm0 2084; AVX2-NEXT: vpsrld $1, %ymm0, %ymm0 2085; AVX2-NEXT: vpsrld $1, %ymm1, %ymm1 2086; AVX2-NEXT: vpsrld $1, %ymm2, %ymm8 2087; AVX2-NEXT: vpsrld $1, %ymm3, %ymm9 2088; AVX2-NEXT: vpsrld $1, %ymm4, %ymm4 2089; AVX2-NEXT: vpsrld $1, %ymm5, %ymm5 2090; AVX2-NEXT: vpsrld $1, %ymm6, %ymm6 2091; AVX2-NEXT: vpsrld $1, %ymm7, %ymm3 2092; AVX2-NEXT: vmovdqa {{.*#+}} ymm2 = [0,1,4,5,8,9,12,13,128,128,128,128,128,128,128,128,0,1,4,5,8,9,12,13,128,128,128,128,128,128,128,128] 2093; AVX2-NEXT: vpshufb %ymm2, %ymm3, %ymm3 2094; AVX2-NEXT: vpermq {{.*#+}} ymm7 = ymm3[0,2,2,3] 2095; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = <0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u> 2096; AVX2-NEXT: vpshufb %xmm3, %xmm7, %xmm7 2097; AVX2-NEXT: vpshufb %ymm2, %ymm6, %ymm6 2098; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm6[0,2,2,3] 2099; AVX2-NEXT: vpshufb %xmm3, %xmm6, %xmm6 2100; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm7[0] 2101; AVX2-NEXT: vpshufb %ymm2, %ymm5, %ymm5 2102; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm5[0,2,2,3] 2103; AVX2-NEXT: vpshufb %xmm3, %xmm5, %xmm5 2104; AVX2-NEXT: vpshufb %ymm2, %ymm4, %ymm4 2105; AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,2,3] 2106; AVX2-NEXT: vpshufb %xmm3, %xmm4, %xmm4 2107; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm5[0] 2108; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm4, %ymm4 2109; AVX2-NEXT: vpshufb %ymm2, %ymm9, %ymm5 2110; AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm5[0,2,2,3] 2111; AVX2-NEXT: vpshufb %xmm3, %xmm5, %xmm5 2112; AVX2-NEXT: vpshufb %ymm2, %ymm8, %ymm6 2113; AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm6[0,2,2,3] 2114; AVX2-NEXT: vpshufb %xmm3, %xmm6, %xmm6 2115; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm5 = xmm6[0],xmm5[0] 2116; AVX2-NEXT: vpshufb %ymm2, %ymm1, %ymm1 2117; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3] 2118; AVX2-NEXT: vpshufb %xmm3, %xmm1, %xmm1 2119; AVX2-NEXT: vpshufb %ymm2, %ymm0, %ymm0 2120; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3] 2121; AVX2-NEXT: vpshufb %xmm3, %xmm0, %xmm0 2122; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] 2123; AVX2-NEXT: vinserti128 $1, %xmm5, %ymm0, %ymm0 2124; AVX2-NEXT: vmovdqu %ymm0, (%rax) 2125; AVX2-NEXT: vmovdqu %ymm4, (%rax) 2126; AVX2-NEXT: vzeroupper 2127; AVX2-NEXT: retq 2128; 2129; AVX512F-LABEL: avg_v64i8_const: 2130; AVX512F: # BB#0: 2131; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero 2132; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero 2133; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero 2134; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm3 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero 2135; AVX512F-NEXT: vbroadcasti64x4 {{.*#+}} zmm4 = mem[0,1,2,3,0,1,2,3] 2136; AVX512F-NEXT: vpaddd %zmm4, %zmm3, %zmm3 2137; AVX512F-NEXT: vpaddd %zmm4, %zmm2, %zmm2 2138; AVX512F-NEXT: vpaddd %zmm4, %zmm1, %zmm1 2139; AVX512F-NEXT: vpaddd %zmm4, %zmm0, %zmm0 2140; AVX512F-NEXT: vpsrld $1, %zmm0, %zmm0 2141; AVX512F-NEXT: vpsrld $1, %zmm1, %zmm1 2142; AVX512F-NEXT: vpsrld $1, %zmm2, %zmm2 2143; AVX512F-NEXT: vpsrld $1, %zmm3, %zmm3 2144; AVX512F-NEXT: vpmovdb %zmm3, %xmm3 2145; AVX512F-NEXT: vpmovdb %zmm2, %xmm2 2146; AVX512F-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2 2147; AVX512F-NEXT: vpmovdb %zmm1, %xmm1 2148; AVX512F-NEXT: vpmovdb %zmm0, %xmm0 2149; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 2150; AVX512F-NEXT: vmovdqu %ymm0, (%rax) 2151; AVX512F-NEXT: vmovdqu %ymm2, (%rax) 2152; AVX512F-NEXT: retq 2153; 2154; AVX512BW-LABEL: avg_v64i8_const: 2155; AVX512BW: # BB#0: 2156; AVX512BW-NEXT: vmovdqu8 (%rdi), %zmm0 2157; AVX512BW-NEXT: vpavgb {{.*}}(%rip), %zmm0, %zmm0 2158; AVX512BW-NEXT: vmovdqu8 %zmm0, (%rax) 2159; AVX512BW-NEXT: retq 2160 %1 = load <64 x i8>, <64 x i8>* %a 2161 %2 = zext <64 x i8> %1 to <64 x i32> 2162 %3 = add nuw nsw <64 x i32> %2, <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8> 2163 %4 = lshr <64 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 2164 %5 = trunc <64 x i32> %4 to <64 x i8> 2165 store <64 x i8> %5, <64 x i8>* undef, align 4 2166 ret void 2167} 2168 2169define void @avg_v4i16_const(<4 x i16>* %a) { 2170; SSE2-LABEL: avg_v4i16_const: 2171; SSE2: # BB#0: 2172; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero 2173; SSE2-NEXT: pavgw {{.*}}(%rip), %xmm0 2174; SSE2-NEXT: movq %xmm0, (%rax) 2175; SSE2-NEXT: retq 2176; 2177; AVX2-LABEL: avg_v4i16_const: 2178; AVX2: # BB#0: 2179; AVX2-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 2180; AVX2-NEXT: vpavgw {{.*}}(%rip), %xmm0, %xmm0 2181; AVX2-NEXT: vmovq %xmm0, (%rax) 2182; AVX2-NEXT: retq 2183; 2184; AVX512F-LABEL: avg_v4i16_const: 2185; AVX512F: # BB#0: 2186; AVX512F-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 2187; AVX512F-NEXT: vpavgw {{.*}}(%rip), %xmm0, %xmm0 2188; AVX512F-NEXT: vmovq %xmm0, (%rax) 2189; AVX512F-NEXT: retq 2190; 2191; AVX512BW-LABEL: avg_v4i16_const: 2192; AVX512BW: # BB#0: 2193; AVX512BW-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 2194; AVX512BW-NEXT: vpavgw {{.*}}(%rip), %xmm0, %xmm0 2195; AVX512BW-NEXT: vmovq %xmm0, (%rax) 2196; AVX512BW-NEXT: retq 2197 %1 = load <4 x i16>, <4 x i16>* %a 2198 %2 = zext <4 x i16> %1 to <4 x i32> 2199 %3 = add nuw nsw <4 x i32> %2, <i32 1, i32 2, i32 3, i32 4> 2200 %4 = lshr <4 x i32> %3, <i32 1, i32 1, i32 1, i32 1> 2201 %5 = trunc <4 x i32> %4 to <4 x i16> 2202 store <4 x i16> %5, <4 x i16>* undef, align 4 2203 ret void 2204} 2205 2206define void @avg_v8i16_const(<8 x i16>* %a) { 2207; SSE2-LABEL: avg_v8i16_const: 2208; SSE2: # BB#0: 2209; SSE2-NEXT: movdqa (%rdi), %xmm0 2210; SSE2-NEXT: pavgw {{.*}}(%rip), %xmm0 2211; SSE2-NEXT: movdqu %xmm0, (%rax) 2212; SSE2-NEXT: retq 2213; 2214; AVX2-LABEL: avg_v8i16_const: 2215; AVX2: # BB#0: 2216; AVX2-NEXT: vmovdqa (%rdi), %xmm0 2217; AVX2-NEXT: vpavgw {{.*}}(%rip), %xmm0, %xmm0 2218; AVX2-NEXT: vmovdqu %xmm0, (%rax) 2219; AVX2-NEXT: retq 2220; 2221; AVX512F-LABEL: avg_v8i16_const: 2222; AVX512F: # BB#0: 2223; AVX512F-NEXT: vmovdqa (%rdi), %xmm0 2224; AVX512F-NEXT: vpavgw {{.*}}(%rip), %xmm0, %xmm0 2225; AVX512F-NEXT: vmovdqu %xmm0, (%rax) 2226; AVX512F-NEXT: retq 2227; 2228; AVX512BW-LABEL: avg_v8i16_const: 2229; AVX512BW: # BB#0: 2230; AVX512BW-NEXT: vmovdqa (%rdi), %xmm0 2231; AVX512BW-NEXT: vpavgw {{.*}}(%rip), %xmm0, %xmm0 2232; AVX512BW-NEXT: vmovdqu %xmm0, (%rax) 2233; AVX512BW-NEXT: retq 2234 %1 = load <8 x i16>, <8 x i16>* %a 2235 %2 = zext <8 x i16> %1 to <8 x i32> 2236 %3 = add nuw nsw <8 x i32> %2, <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8> 2237 %4 = lshr <8 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 2238 %5 = trunc <8 x i32> %4 to <8 x i16> 2239 store <8 x i16> %5, <8 x i16>* undef, align 4 2240 ret void 2241} 2242 2243define void @avg_v16i16_const(<16 x i16>* %a) { 2244; SSE2-LABEL: avg_v16i16_const: 2245; SSE2: # BB#0: 2246; SSE2-NEXT: movdqa (%rdi), %xmm3 2247; SSE2-NEXT: movdqa 16(%rdi), %xmm0 2248; SSE2-NEXT: pxor %xmm4, %xmm4 2249; SSE2-NEXT: movdqa %xmm0, %xmm1 2250; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3] 2251; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7] 2252; SSE2-NEXT: movdqa %xmm3, %xmm2 2253; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3] 2254; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7] 2255; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [5,6,7,8] 2256; SSE2-NEXT: paddd %xmm4, %xmm3 2257; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [1,2,3,4] 2258; SSE2-NEXT: paddd %xmm5, %xmm2 2259; SSE2-NEXT: paddd %xmm4, %xmm0 2260; SSE2-NEXT: paddd %xmm5, %xmm1 2261; SSE2-NEXT: psrld $1, %xmm1 2262; SSE2-NEXT: psrld $1, %xmm0 2263; SSE2-NEXT: psrld $1, %xmm2 2264; SSE2-NEXT: psrld $1, %xmm3 2265; SSE2-NEXT: pslld $16, %xmm3 2266; SSE2-NEXT: psrad $16, %xmm3 2267; SSE2-NEXT: pslld $16, %xmm2 2268; SSE2-NEXT: psrad $16, %xmm2 2269; SSE2-NEXT: packssdw %xmm3, %xmm2 2270; SSE2-NEXT: pslld $16, %xmm0 2271; SSE2-NEXT: psrad $16, %xmm0 2272; SSE2-NEXT: pslld $16, %xmm1 2273; SSE2-NEXT: psrad $16, %xmm1 2274; SSE2-NEXT: packssdw %xmm0, %xmm1 2275; SSE2-NEXT: movdqu %xmm1, (%rax) 2276; SSE2-NEXT: movdqu %xmm2, (%rax) 2277; SSE2-NEXT: retq 2278; 2279; AVX2-LABEL: avg_v16i16_const: 2280; AVX2: # BB#0: 2281; AVX2-NEXT: vmovdqa (%rdi), %ymm0 2282; AVX2-NEXT: vpavgw {{.*}}(%rip), %ymm0, %ymm0 2283; AVX2-NEXT: vmovdqu %ymm0, (%rax) 2284; AVX2-NEXT: vzeroupper 2285; AVX2-NEXT: retq 2286; 2287; AVX512F-LABEL: avg_v16i16_const: 2288; AVX512F: # BB#0: 2289; AVX512F-NEXT: vmovdqa (%rdi), %ymm0 2290; AVX512F-NEXT: vpavgw {{.*}}(%rip), %ymm0, %ymm0 2291; AVX512F-NEXT: vmovdqu %ymm0, (%rax) 2292; AVX512F-NEXT: retq 2293; 2294; AVX512BW-LABEL: avg_v16i16_const: 2295; AVX512BW: # BB#0: 2296; AVX512BW-NEXT: vmovdqa (%rdi), %ymm0 2297; AVX512BW-NEXT: vpavgw {{.*}}(%rip), %ymm0, %ymm0 2298; AVX512BW-NEXT: vmovdqu %ymm0, (%rax) 2299; AVX512BW-NEXT: retq 2300 %1 = load <16 x i16>, <16 x i16>* %a 2301 %2 = zext <16 x i16> %1 to <16 x i32> 2302 %3 = add nuw nsw <16 x i32> %2, <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8> 2303 %4 = lshr <16 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 2304 %5 = trunc <16 x i32> %4 to <16 x i16> 2305 store <16 x i16> %5, <16 x i16>* undef, align 4 2306 ret void 2307} 2308 2309define void @avg_v32i16_const(<32 x i16>* %a) { 2310; SSE2-LABEL: avg_v32i16_const: 2311; SSE2: # BB#0: 2312; SSE2-NEXT: movdqa (%rdi), %xmm7 2313; SSE2-NEXT: movdqa 16(%rdi), %xmm6 2314; SSE2-NEXT: movdqa 32(%rdi), %xmm4 2315; SSE2-NEXT: movdqa 48(%rdi), %xmm0 2316; SSE2-NEXT: pxor %xmm8, %xmm8 2317; SSE2-NEXT: movdqa %xmm0, %xmm1 2318; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1],xmm1[2],xmm8[2],xmm1[3],xmm8[3] 2319; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm8[4],xmm0[5],xmm8[5],xmm0[6],xmm8[6],xmm0[7],xmm8[7] 2320; SSE2-NEXT: movdqa %xmm4, %xmm2 2321; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm8[0],xmm2[1],xmm8[1],xmm2[2],xmm8[2],xmm2[3],xmm8[3] 2322; SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm8[4],xmm4[5],xmm8[5],xmm4[6],xmm8[6],xmm4[7],xmm8[7] 2323; SSE2-NEXT: movdqa %xmm6, %xmm3 2324; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm8[0],xmm3[1],xmm8[1],xmm3[2],xmm8[2],xmm3[3],xmm8[3] 2325; SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm8[4],xmm6[5],xmm8[5],xmm6[6],xmm8[6],xmm6[7],xmm8[7] 2326; SSE2-NEXT: movdqa %xmm7, %xmm5 2327; SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm8[0],xmm5[1],xmm8[1],xmm5[2],xmm8[2],xmm5[3],xmm8[3] 2328; SSE2-NEXT: punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm8[4],xmm7[5],xmm8[5],xmm7[6],xmm8[6],xmm7[7],xmm8[7] 2329; SSE2-NEXT: movdqa {{.*#+}} xmm8 = [5,6,7,8] 2330; SSE2-NEXT: paddd %xmm8, %xmm7 2331; SSE2-NEXT: movdqa {{.*#+}} xmm9 = [1,2,3,4] 2332; SSE2-NEXT: paddd %xmm9, %xmm5 2333; SSE2-NEXT: paddd %xmm8, %xmm6 2334; SSE2-NEXT: paddd %xmm9, %xmm3 2335; SSE2-NEXT: paddd %xmm8, %xmm4 2336; SSE2-NEXT: paddd %xmm9, %xmm2 2337; SSE2-NEXT: paddd %xmm8, %xmm0 2338; SSE2-NEXT: paddd %xmm9, %xmm1 2339; SSE2-NEXT: psrld $1, %xmm1 2340; SSE2-NEXT: psrld $1, %xmm0 2341; SSE2-NEXT: psrld $1, %xmm2 2342; SSE2-NEXT: psrld $1, %xmm4 2343; SSE2-NEXT: psrld $1, %xmm3 2344; SSE2-NEXT: psrld $1, %xmm6 2345; SSE2-NEXT: psrld $1, %xmm5 2346; SSE2-NEXT: psrld $1, %xmm7 2347; SSE2-NEXT: pslld $16, %xmm7 2348; SSE2-NEXT: psrad $16, %xmm7 2349; SSE2-NEXT: pslld $16, %xmm5 2350; SSE2-NEXT: psrad $16, %xmm5 2351; SSE2-NEXT: packssdw %xmm7, %xmm5 2352; SSE2-NEXT: pslld $16, %xmm6 2353; SSE2-NEXT: psrad $16, %xmm6 2354; SSE2-NEXT: pslld $16, %xmm3 2355; SSE2-NEXT: psrad $16, %xmm3 2356; SSE2-NEXT: packssdw %xmm6, %xmm3 2357; SSE2-NEXT: pslld $16, %xmm4 2358; SSE2-NEXT: psrad $16, %xmm4 2359; SSE2-NEXT: pslld $16, %xmm2 2360; SSE2-NEXT: psrad $16, %xmm2 2361; SSE2-NEXT: packssdw %xmm4, %xmm2 2362; SSE2-NEXT: pslld $16, %xmm0 2363; SSE2-NEXT: psrad $16, %xmm0 2364; SSE2-NEXT: pslld $16, %xmm1 2365; SSE2-NEXT: psrad $16, %xmm1 2366; SSE2-NEXT: packssdw %xmm0, %xmm1 2367; SSE2-NEXT: movdqu %xmm1, (%rax) 2368; SSE2-NEXT: movdqu %xmm2, (%rax) 2369; SSE2-NEXT: movdqu %xmm3, (%rax) 2370; SSE2-NEXT: movdqu %xmm5, (%rax) 2371; SSE2-NEXT: retq 2372; 2373; AVX2-LABEL: avg_v32i16_const: 2374; AVX2: # BB#0: 2375; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 2376; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 2377; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 2378; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 2379; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [1,2,3,4,5,6,7,8] 2380; AVX2-NEXT: vpaddd %ymm4, %ymm3, %ymm3 2381; AVX2-NEXT: vpaddd %ymm4, %ymm2, %ymm2 2382; AVX2-NEXT: vpaddd %ymm4, %ymm1, %ymm1 2383; AVX2-NEXT: vpaddd %ymm4, %ymm0, %ymm0 2384; AVX2-NEXT: vpsrld $1, %ymm0, %ymm0 2385; AVX2-NEXT: vpsrld $1, %ymm1, %ymm1 2386; AVX2-NEXT: vpsrld $1, %ymm2, %ymm2 2387; AVX2-NEXT: vpsrld $1, %ymm3, %ymm3 2388; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,4,5,8,9,12,13,128,128,128,128,128,128,128,128,0,1,4,5,8,9,12,13,128,128,128,128,128,128,128,128] 2389; AVX2-NEXT: vpshufb %ymm4, %ymm3, %ymm3 2390; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,2,3] 2391; AVX2-NEXT: vpshufb %ymm4, %ymm2, %ymm2 2392; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3] 2393; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2 2394; AVX2-NEXT: vpshufb %ymm4, %ymm1, %ymm1 2395; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,2,3] 2396; AVX2-NEXT: vpshufb %ymm4, %ymm0, %ymm0 2397; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3] 2398; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 2399; AVX2-NEXT: vmovdqu %ymm0, (%rax) 2400; AVX2-NEXT: vmovdqu %ymm2, (%rax) 2401; AVX2-NEXT: vzeroupper 2402; AVX2-NEXT: retq 2403; 2404; AVX512F-LABEL: avg_v32i16_const: 2405; AVX512F: # BB#0: 2406; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero 2407; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero 2408; AVX512F-NEXT: vbroadcasti64x4 {{.*#+}} zmm2 = mem[0,1,2,3,0,1,2,3] 2409; AVX512F-NEXT: vpaddd %zmm2, %zmm1, %zmm1 2410; AVX512F-NEXT: vpaddd %zmm2, %zmm0, %zmm0 2411; AVX512F-NEXT: vpsrld $1, %zmm0, %zmm0 2412; AVX512F-NEXT: vpsrld $1, %zmm1, %zmm1 2413; AVX512F-NEXT: vpmovdw %zmm1, (%rax) 2414; AVX512F-NEXT: vpmovdw %zmm0, (%rax) 2415; AVX512F-NEXT: retq 2416; 2417; AVX512BW-LABEL: avg_v32i16_const: 2418; AVX512BW: # BB#0: 2419; AVX512BW-NEXT: vmovdqu16 (%rdi), %zmm0 2420; AVX512BW-NEXT: vpavgw {{.*}}(%rip), %zmm0, %zmm0 2421; AVX512BW-NEXT: vmovdqu16 %zmm0, (%rax) 2422; AVX512BW-NEXT: retq 2423 %1 = load <32 x i16>, <32 x i16>* %a 2424 %2 = zext <32 x i16> %1 to <32 x i32> 2425 %3 = add nuw nsw <32 x i32> %2, <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8> 2426 %4 = lshr <32 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 2427 %5 = trunc <32 x i32> %4 to <32 x i16> 2428 store <32 x i16> %5, <32 x i16>* undef, align 4 2429 ret void 2430} 2431