1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py 2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=SSE2 3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=AVX --check-prefix=AVX1 4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2 5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512F 6; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512BW 7 8define void @avg_v4i8(ptr %a, ptr %b) nounwind { 9; SSE2-LABEL: avg_v4i8: 10; SSE2: # %bb.0: 11; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero 12; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero 13; SSE2-NEXT: pavgb %xmm0, %xmm1 14; SSE2-NEXT: movd %xmm1, (%rax) 15; SSE2-NEXT: retq 16; 17; AVX-LABEL: avg_v4i8: 18; AVX: # %bb.0: 19; AVX-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero 20; AVX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero 21; AVX-NEXT: vpavgb %xmm0, %xmm1, %xmm0 22; AVX-NEXT: vmovd %xmm0, (%rax) 23; AVX-NEXT: retq 24 %1 = load <4 x i8>, ptr %a 25 %2 = load <4 x i8>, ptr %b 26 %3 = zext <4 x i8> %1 to <4 x i32> 27 %4 = zext <4 x i8> %2 to <4 x i32> 28 %5 = add nuw nsw <4 x i32> %3, <i32 1, i32 1, i32 1, i32 1> 29 %6 = add nuw nsw <4 x i32> %5, %4 30 %7 = lshr <4 x i32> %6, <i32 1, i32 1, i32 1, i32 1> 31 %8 = trunc <4 x i32> %7 to <4 x i8> 32 store <4 x i8> %8, ptr undef, align 4 33 ret void 34} 35 36define void @avg_v8i8(ptr %a, ptr %b) nounwind { 37; SSE2-LABEL: avg_v8i8: 38; SSE2: # %bb.0: 39; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero 40; SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero 41; SSE2-NEXT: pavgb %xmm0, %xmm1 42; SSE2-NEXT: movq %xmm1, (%rax) 43; SSE2-NEXT: retq 44; 45; AVX-LABEL: avg_v8i8: 46; AVX: # %bb.0: 47; AVX-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 48; AVX-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero 49; AVX-NEXT: vpavgb %xmm0, %xmm1, %xmm0 50; AVX-NEXT: vmovq %xmm0, (%rax) 51; AVX-NEXT: retq 52 %1 = load <8 x i8>, ptr %a 53 %2 = load <8 x i8>, ptr %b 54 %3 = zext <8 x i8> %1 to <8 x i32> 55 %4 = zext <8 x i8> %2 to <8 x i32> 56 %5 = add nuw nsw <8 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 57 %6 = add nuw nsw <8 x i32> %5, %4 58 %7 = lshr <8 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 59 %8 = trunc <8 x i32> %7 to <8 x i8> 60 store <8 x i8> %8, ptr undef, align 4 61 ret void 62} 63 64define void @avg_v16i8(ptr %a, ptr %b) nounwind { 65; SSE2-LABEL: avg_v16i8: 66; SSE2: # %bb.0: 67; SSE2-NEXT: movdqa (%rdi), %xmm0 68; SSE2-NEXT: pavgb (%rsi), %xmm0 69; SSE2-NEXT: movdqu %xmm0, (%rax) 70; SSE2-NEXT: retq 71; 72; AVX-LABEL: avg_v16i8: 73; AVX: # %bb.0: 74; AVX-NEXT: vmovdqa (%rdi), %xmm0 75; AVX-NEXT: vpavgb (%rsi), %xmm0, %xmm0 76; AVX-NEXT: vmovdqu %xmm0, (%rax) 77; AVX-NEXT: retq 78 %1 = load <16 x i8>, ptr %a 79 %2 = load <16 x i8>, ptr %b 80 %3 = zext <16 x i8> %1 to <16 x i32> 81 %4 = zext <16 x i8> %2 to <16 x i32> 82 %5 = add nuw nsw <16 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 83 %6 = add nuw nsw <16 x i32> %5, %4 84 %7 = lshr <16 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 85 %8 = trunc <16 x i32> %7 to <16 x i8> 86 store <16 x i8> %8, ptr undef, align 4 87 ret void 88} 89 90define void @avg_v24i8(ptr %a, ptr %b) nounwind { 91; SSE2-LABEL: avg_v24i8: 92; SSE2: # %bb.0: 93; SSE2-NEXT: movdqa (%rsi), %xmm0 94; SSE2-NEXT: movdqa 16(%rsi), %xmm1 95; SSE2-NEXT: pavgb (%rdi), %xmm0 96; SSE2-NEXT: pavgb 16(%rdi), %xmm1 97; SSE2-NEXT: movq %xmm1, (%rax) 98; SSE2-NEXT: movdqu %xmm0, (%rax) 99; SSE2-NEXT: retq 100; 101; AVX1-LABEL: avg_v24i8: 102; AVX1: # %bb.0: 103; AVX1-NEXT: vmovdqa (%rsi), %xmm0 104; AVX1-NEXT: vmovdqa 16(%rsi), %xmm1 105; AVX1-NEXT: vpavgb (%rdi), %xmm0, %xmm0 106; AVX1-NEXT: vpavgb 16(%rdi), %xmm1, %xmm1 107; AVX1-NEXT: vmovq %xmm1, (%rax) 108; AVX1-NEXT: vmovdqu %xmm0, (%rax) 109; AVX1-NEXT: retq 110; 111; AVX2-LABEL: avg_v24i8: 112; AVX2: # %bb.0: 113; AVX2-NEXT: vmovdqa (%rsi), %ymm0 114; AVX2-NEXT: vpavgb (%rdi), %ymm0, %ymm0 115; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 116; AVX2-NEXT: vmovq %xmm1, (%rax) 117; AVX2-NEXT: vmovdqu %xmm0, (%rax) 118; AVX2-NEXT: vzeroupper 119; AVX2-NEXT: retq 120; 121; AVX512-LABEL: avg_v24i8: 122; AVX512: # %bb.0: 123; AVX512-NEXT: vmovdqa (%rsi), %ymm0 124; AVX512-NEXT: vpavgb (%rdi), %ymm0, %ymm0 125; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1 126; AVX512-NEXT: vmovq %xmm1, (%rax) 127; AVX512-NEXT: vmovdqu %xmm0, (%rax) 128; AVX512-NEXT: vzeroupper 129; AVX512-NEXT: retq 130 %1 = load <24 x i8>, ptr %a 131 %2 = load <24 x i8>, ptr %b 132 %3 = zext <24 x i8> %1 to <24 x i32> 133 %4 = zext <24 x i8> %2 to <24 x i32> 134 %5 = add nuw nsw <24 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 135 %6 = add nuw nsw <24 x i32> %5, %4 136 %7 = lshr <24 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 137 %8 = trunc <24 x i32> %7 to <24 x i8> 138 store <24 x i8> %8, ptr undef, align 4 139 ret void 140} 141 142define void @avg_v32i8(ptr %a, ptr %b) nounwind { 143; SSE2-LABEL: avg_v32i8: 144; SSE2: # %bb.0: 145; SSE2-NEXT: movdqa (%rsi), %xmm0 146; SSE2-NEXT: movdqa 16(%rsi), %xmm1 147; SSE2-NEXT: pavgb (%rdi), %xmm0 148; SSE2-NEXT: pavgb 16(%rdi), %xmm1 149; SSE2-NEXT: movdqu %xmm1, (%rax) 150; SSE2-NEXT: movdqu %xmm0, (%rax) 151; SSE2-NEXT: retq 152; 153; AVX1-LABEL: avg_v32i8: 154; AVX1: # %bb.0: 155; AVX1-NEXT: vmovdqa (%rdi), %xmm0 156; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 157; AVX1-NEXT: vpavgb (%rsi), %xmm0, %xmm0 158; AVX1-NEXT: vpavgb 16(%rsi), %xmm1, %xmm1 159; AVX1-NEXT: vmovdqu %xmm1, (%rax) 160; AVX1-NEXT: vmovdqu %xmm0, (%rax) 161; AVX1-NEXT: retq 162; 163; AVX2-LABEL: avg_v32i8: 164; AVX2: # %bb.0: 165; AVX2-NEXT: vmovdqa (%rdi), %ymm0 166; AVX2-NEXT: vpavgb (%rsi), %ymm0, %ymm0 167; AVX2-NEXT: vmovdqu %ymm0, (%rax) 168; AVX2-NEXT: vzeroupper 169; AVX2-NEXT: retq 170; 171; AVX512-LABEL: avg_v32i8: 172; AVX512: # %bb.0: 173; AVX512-NEXT: vmovdqa (%rdi), %ymm0 174; AVX512-NEXT: vpavgb (%rsi), %ymm0, %ymm0 175; AVX512-NEXT: vmovdqu %ymm0, (%rax) 176; AVX512-NEXT: vzeroupper 177; AVX512-NEXT: retq 178 %1 = load <32 x i8>, ptr %a 179 %2 = load <32 x i8>, ptr %b 180 %3 = zext <32 x i8> %1 to <32 x i32> 181 %4 = zext <32 x i8> %2 to <32 x i32> 182 %5 = add nuw nsw <32 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 183 %6 = add nuw nsw <32 x i32> %5, %4 184 %7 = lshr <32 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 185 %8 = trunc <32 x i32> %7 to <32 x i8> 186 store <32 x i8> %8, ptr undef, align 4 187 ret void 188} 189 190define void @avg_v48i8(ptr %a, ptr %b) nounwind { 191; SSE2-LABEL: avg_v48i8: 192; SSE2: # %bb.0: 193; SSE2-NEXT: movdqa (%rsi), %xmm0 194; SSE2-NEXT: movdqa 16(%rsi), %xmm1 195; SSE2-NEXT: movdqa 32(%rsi), %xmm2 196; SSE2-NEXT: pavgb (%rdi), %xmm0 197; SSE2-NEXT: pavgb 16(%rdi), %xmm1 198; SSE2-NEXT: pavgb 32(%rdi), %xmm2 199; SSE2-NEXT: movdqu %xmm2, (%rax) 200; SSE2-NEXT: movdqu %xmm1, (%rax) 201; SSE2-NEXT: movdqu %xmm0, (%rax) 202; SSE2-NEXT: retq 203; 204; AVX1-LABEL: avg_v48i8: 205; AVX1: # %bb.0: 206; AVX1-NEXT: vmovdqa (%rsi), %xmm0 207; AVX1-NEXT: vmovdqa 16(%rsi), %xmm1 208; AVX1-NEXT: vmovdqa 32(%rsi), %xmm2 209; AVX1-NEXT: vpavgb 32(%rdi), %xmm2, %xmm2 210; AVX1-NEXT: vpavgb (%rdi), %xmm0, %xmm0 211; AVX1-NEXT: vpavgb 16(%rdi), %xmm1, %xmm1 212; AVX1-NEXT: vmovdqu %xmm1, (%rax) 213; AVX1-NEXT: vmovdqu %xmm0, (%rax) 214; AVX1-NEXT: vmovdqu %xmm2, (%rax) 215; AVX1-NEXT: retq 216; 217; AVX2-LABEL: avg_v48i8: 218; AVX2: # %bb.0: 219; AVX2-NEXT: vmovdqa (%rsi), %ymm0 220; AVX2-NEXT: vpavgb (%rdi), %ymm0, %ymm0 221; AVX2-NEXT: vmovdqa 32(%rsi), %xmm1 222; AVX2-NEXT: vpavgb 32(%rdi), %xmm1, %xmm1 223; AVX2-NEXT: vmovdqu %xmm1, (%rax) 224; AVX2-NEXT: vmovdqu %ymm0, (%rax) 225; AVX2-NEXT: vzeroupper 226; AVX2-NEXT: retq 227; 228; AVX512F-LABEL: avg_v48i8: 229; AVX512F: # %bb.0: 230; AVX512F-NEXT: vmovdqa (%rsi), %ymm0 231; AVX512F-NEXT: vpavgb (%rdi), %ymm0, %ymm0 232; AVX512F-NEXT: vmovdqa 32(%rsi), %xmm1 233; AVX512F-NEXT: vpavgb 32(%rdi), %xmm1, %xmm1 234; AVX512F-NEXT: vmovdqu %xmm1, (%rax) 235; AVX512F-NEXT: vmovdqu %ymm0, (%rax) 236; AVX512F-NEXT: vzeroupper 237; AVX512F-NEXT: retq 238; 239; AVX512BW-LABEL: avg_v48i8: 240; AVX512BW: # %bb.0: 241; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm0 242; AVX512BW-NEXT: vpavgb (%rdi), %zmm0, %zmm0 243; AVX512BW-NEXT: vextracti32x4 $2, %zmm0, (%rax) 244; AVX512BW-NEXT: vmovdqu %ymm0, (%rax) 245; AVX512BW-NEXT: vzeroupper 246; AVX512BW-NEXT: retq 247 %1 = load <48 x i8>, ptr %a 248 %2 = load <48 x i8>, ptr %b 249 %3 = zext <48 x i8> %1 to <48 x i32> 250 %4 = zext <48 x i8> %2 to <48 x i32> 251 %5 = add nuw nsw <48 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 252 %6 = add nuw nsw <48 x i32> %5, %4 253 %7 = lshr <48 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 254 %8 = trunc <48 x i32> %7 to <48 x i8> 255 store <48 x i8> %8, ptr undef, align 4 256 ret void 257} 258 259define void @avg_v64i8(ptr %a, ptr %b) nounwind { 260; SSE2-LABEL: avg_v64i8: 261; SSE2: # %bb.0: 262; SSE2-NEXT: movdqa (%rsi), %xmm0 263; SSE2-NEXT: movdqa 16(%rsi), %xmm1 264; SSE2-NEXT: movdqa 32(%rsi), %xmm2 265; SSE2-NEXT: movdqa 48(%rsi), %xmm3 266; SSE2-NEXT: pavgb (%rdi), %xmm0 267; SSE2-NEXT: pavgb 16(%rdi), %xmm1 268; SSE2-NEXT: pavgb 32(%rdi), %xmm2 269; SSE2-NEXT: pavgb 48(%rdi), %xmm3 270; SSE2-NEXT: movdqu %xmm3, (%rax) 271; SSE2-NEXT: movdqu %xmm2, (%rax) 272; SSE2-NEXT: movdqu %xmm1, (%rax) 273; SSE2-NEXT: movdqu %xmm0, (%rax) 274; SSE2-NEXT: retq 275; 276; AVX1-LABEL: avg_v64i8: 277; AVX1: # %bb.0: 278; AVX1-NEXT: vmovdqa (%rsi), %xmm0 279; AVX1-NEXT: vmovdqa 16(%rsi), %xmm1 280; AVX1-NEXT: vmovdqa 32(%rsi), %xmm2 281; AVX1-NEXT: vmovdqa 48(%rsi), %xmm3 282; AVX1-NEXT: vpavgb (%rdi), %xmm0, %xmm0 283; AVX1-NEXT: vpavgb 16(%rdi), %xmm1, %xmm1 284; AVX1-NEXT: vpavgb 32(%rdi), %xmm2, %xmm2 285; AVX1-NEXT: vpavgb 48(%rdi), %xmm3, %xmm3 286; AVX1-NEXT: vmovdqu %xmm3, (%rax) 287; AVX1-NEXT: vmovdqu %xmm2, (%rax) 288; AVX1-NEXT: vmovdqu %xmm1, (%rax) 289; AVX1-NEXT: vmovdqu %xmm0, (%rax) 290; AVX1-NEXT: retq 291; 292; AVX2-LABEL: avg_v64i8: 293; AVX2: # %bb.0: 294; AVX2-NEXT: vmovdqa (%rsi), %ymm0 295; AVX2-NEXT: vmovdqa 32(%rsi), %ymm1 296; AVX2-NEXT: vpavgb (%rdi), %ymm0, %ymm0 297; AVX2-NEXT: vpavgb 32(%rdi), %ymm1, %ymm1 298; AVX2-NEXT: vmovdqu %ymm1, (%rax) 299; AVX2-NEXT: vmovdqu %ymm0, (%rax) 300; AVX2-NEXT: vzeroupper 301; AVX2-NEXT: retq 302; 303; AVX512F-LABEL: avg_v64i8: 304; AVX512F: # %bb.0: 305; AVX512F-NEXT: vmovdqa (%rdi), %ymm0 306; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm1 307; AVX512F-NEXT: vpavgb (%rsi), %ymm0, %ymm0 308; AVX512F-NEXT: vpavgb 32(%rsi), %ymm1, %ymm1 309; AVX512F-NEXT: vmovdqu %ymm1, (%rax) 310; AVX512F-NEXT: vmovdqu %ymm0, (%rax) 311; AVX512F-NEXT: vzeroupper 312; AVX512F-NEXT: retq 313; 314; AVX512BW-LABEL: avg_v64i8: 315; AVX512BW: # %bb.0: 316; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0 317; AVX512BW-NEXT: vpavgb (%rsi), %zmm0, %zmm0 318; AVX512BW-NEXT: vmovdqu64 %zmm0, (%rax) 319; AVX512BW-NEXT: vzeroupper 320; AVX512BW-NEXT: retq 321 %1 = load <64 x i8>, ptr %a 322 %2 = load <64 x i8>, ptr %b 323 %3 = zext <64 x i8> %1 to <64 x i32> 324 %4 = zext <64 x i8> %2 to <64 x i32> 325 %5 = add nuw nsw <64 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 326 %6 = add nuw nsw <64 x i32> %5, %4 327 %7 = lshr <64 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 328 %8 = trunc <64 x i32> %7 to <64 x i8> 329 store <64 x i8> %8, ptr undef, align 4 330 ret void 331} 332 333define void @avg_v4i16(ptr %a, ptr %b) nounwind { 334; SSE2-LABEL: avg_v4i16: 335; SSE2: # %bb.0: 336; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero 337; SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero 338; SSE2-NEXT: pavgw %xmm0, %xmm1 339; SSE2-NEXT: movq %xmm1, (%rax) 340; SSE2-NEXT: retq 341; 342; AVX-LABEL: avg_v4i16: 343; AVX: # %bb.0: 344; AVX-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 345; AVX-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero 346; AVX-NEXT: vpavgw %xmm0, %xmm1, %xmm0 347; AVX-NEXT: vmovq %xmm0, (%rax) 348; AVX-NEXT: retq 349 %1 = load <4 x i16>, ptr %a 350 %2 = load <4 x i16>, ptr %b 351 %3 = zext <4 x i16> %1 to <4 x i32> 352 %4 = zext <4 x i16> %2 to <4 x i32> 353 %5 = add nuw nsw <4 x i32> %3, <i32 1, i32 1, i32 1, i32 1> 354 %6 = add nuw nsw <4 x i32> %5, %4 355 %7 = lshr <4 x i32> %6, <i32 1, i32 1, i32 1, i32 1> 356 %8 = trunc <4 x i32> %7 to <4 x i16> 357 store <4 x i16> %8, ptr undef, align 4 358 ret void 359} 360 361define void @avg_v8i16(ptr %a, ptr %b) nounwind { 362; SSE2-LABEL: avg_v8i16: 363; SSE2: # %bb.0: 364; SSE2-NEXT: movdqa (%rdi), %xmm0 365; SSE2-NEXT: pavgw (%rsi), %xmm0 366; SSE2-NEXT: movdqu %xmm0, (%rax) 367; SSE2-NEXT: retq 368; 369; AVX-LABEL: avg_v8i16: 370; AVX: # %bb.0: 371; AVX-NEXT: vmovdqa (%rdi), %xmm0 372; AVX-NEXT: vpavgw (%rsi), %xmm0, %xmm0 373; AVX-NEXT: vmovdqu %xmm0, (%rax) 374; AVX-NEXT: retq 375 %1 = load <8 x i16>, ptr %a 376 %2 = load <8 x i16>, ptr %b 377 %3 = zext <8 x i16> %1 to <8 x i32> 378 %4 = zext <8 x i16> %2 to <8 x i32> 379 %5 = add nuw nsw <8 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 380 %6 = add nuw nsw <8 x i32> %5, %4 381 %7 = lshr <8 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 382 %8 = trunc <8 x i32> %7 to <8 x i16> 383 store <8 x i16> %8, ptr undef, align 4 384 ret void 385} 386 387define void @avg_v16i16(ptr %a, ptr %b) nounwind { 388; SSE2-LABEL: avg_v16i16: 389; SSE2: # %bb.0: 390; SSE2-NEXT: movdqa (%rsi), %xmm0 391; SSE2-NEXT: movdqa 16(%rsi), %xmm1 392; SSE2-NEXT: pavgw (%rdi), %xmm0 393; SSE2-NEXT: pavgw 16(%rdi), %xmm1 394; SSE2-NEXT: movdqu %xmm1, (%rax) 395; SSE2-NEXT: movdqu %xmm0, (%rax) 396; SSE2-NEXT: retq 397; 398; AVX1-LABEL: avg_v16i16: 399; AVX1: # %bb.0: 400; AVX1-NEXT: vmovdqa (%rdi), %xmm0 401; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 402; AVX1-NEXT: vpavgw (%rsi), %xmm0, %xmm0 403; AVX1-NEXT: vpavgw 16(%rsi), %xmm1, %xmm1 404; AVX1-NEXT: vmovdqu %xmm1, (%rax) 405; AVX1-NEXT: vmovdqu %xmm0, (%rax) 406; AVX1-NEXT: retq 407; 408; AVX2-LABEL: avg_v16i16: 409; AVX2: # %bb.0: 410; AVX2-NEXT: vmovdqa (%rdi), %ymm0 411; AVX2-NEXT: vpavgw (%rsi), %ymm0, %ymm0 412; AVX2-NEXT: vmovdqu %ymm0, (%rax) 413; AVX2-NEXT: vzeroupper 414; AVX2-NEXT: retq 415; 416; AVX512-LABEL: avg_v16i16: 417; AVX512: # %bb.0: 418; AVX512-NEXT: vmovdqa (%rdi), %ymm0 419; AVX512-NEXT: vpavgw (%rsi), %ymm0, %ymm0 420; AVX512-NEXT: vmovdqu %ymm0, (%rax) 421; AVX512-NEXT: vzeroupper 422; AVX512-NEXT: retq 423 %1 = load <16 x i16>, ptr %a 424 %2 = load <16 x i16>, ptr %b 425 %3 = zext <16 x i16> %1 to <16 x i32> 426 %4 = zext <16 x i16> %2 to <16 x i32> 427 %5 = add nuw nsw <16 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 428 %6 = add nuw nsw <16 x i32> %5, %4 429 %7 = lshr <16 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 430 %8 = trunc <16 x i32> %7 to <16 x i16> 431 store <16 x i16> %8, ptr undef, align 4 432 ret void 433} 434 435define void @avg_v32i16(ptr %a, ptr %b) nounwind { 436; SSE2-LABEL: avg_v32i16: 437; SSE2: # %bb.0: 438; SSE2-NEXT: movdqa (%rsi), %xmm0 439; SSE2-NEXT: movdqa 16(%rsi), %xmm1 440; SSE2-NEXT: movdqa 32(%rsi), %xmm2 441; SSE2-NEXT: movdqa 48(%rsi), %xmm3 442; SSE2-NEXT: pavgw (%rdi), %xmm0 443; SSE2-NEXT: pavgw 16(%rdi), %xmm1 444; SSE2-NEXT: pavgw 32(%rdi), %xmm2 445; SSE2-NEXT: pavgw 48(%rdi), %xmm3 446; SSE2-NEXT: movdqu %xmm3, (%rax) 447; SSE2-NEXT: movdqu %xmm2, (%rax) 448; SSE2-NEXT: movdqu %xmm1, (%rax) 449; SSE2-NEXT: movdqu %xmm0, (%rax) 450; SSE2-NEXT: retq 451; 452; AVX1-LABEL: avg_v32i16: 453; AVX1: # %bb.0: 454; AVX1-NEXT: vmovdqa (%rsi), %xmm0 455; AVX1-NEXT: vmovdqa 16(%rsi), %xmm1 456; AVX1-NEXT: vmovdqa 32(%rsi), %xmm2 457; AVX1-NEXT: vmovdqa 48(%rsi), %xmm3 458; AVX1-NEXT: vpavgw (%rdi), %xmm0, %xmm0 459; AVX1-NEXT: vpavgw 16(%rdi), %xmm1, %xmm1 460; AVX1-NEXT: vpavgw 32(%rdi), %xmm2, %xmm2 461; AVX1-NEXT: vpavgw 48(%rdi), %xmm3, %xmm3 462; AVX1-NEXT: vmovdqu %xmm3, (%rax) 463; AVX1-NEXT: vmovdqu %xmm2, (%rax) 464; AVX1-NEXT: vmovdqu %xmm1, (%rax) 465; AVX1-NEXT: vmovdqu %xmm0, (%rax) 466; AVX1-NEXT: retq 467; 468; AVX2-LABEL: avg_v32i16: 469; AVX2: # %bb.0: 470; AVX2-NEXT: vmovdqa (%rsi), %ymm0 471; AVX2-NEXT: vmovdqa 32(%rsi), %ymm1 472; AVX2-NEXT: vpavgw (%rdi), %ymm0, %ymm0 473; AVX2-NEXT: vpavgw 32(%rdi), %ymm1, %ymm1 474; AVX2-NEXT: vmovdqu %ymm1, (%rax) 475; AVX2-NEXT: vmovdqu %ymm0, (%rax) 476; AVX2-NEXT: vzeroupper 477; AVX2-NEXT: retq 478; 479; AVX512F-LABEL: avg_v32i16: 480; AVX512F: # %bb.0: 481; AVX512F-NEXT: vmovdqa (%rdi), %ymm0 482; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm1 483; AVX512F-NEXT: vpavgw (%rsi), %ymm0, %ymm0 484; AVX512F-NEXT: vpavgw 32(%rsi), %ymm1, %ymm1 485; AVX512F-NEXT: vmovdqu %ymm1, (%rax) 486; AVX512F-NEXT: vmovdqu %ymm0, (%rax) 487; AVX512F-NEXT: vzeroupper 488; AVX512F-NEXT: retq 489; 490; AVX512BW-LABEL: avg_v32i16: 491; AVX512BW: # %bb.0: 492; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0 493; AVX512BW-NEXT: vpavgw (%rsi), %zmm0, %zmm0 494; AVX512BW-NEXT: vmovdqu64 %zmm0, (%rax) 495; AVX512BW-NEXT: vzeroupper 496; AVX512BW-NEXT: retq 497 %1 = load <32 x i16>, ptr %a 498 %2 = load <32 x i16>, ptr %b 499 %3 = zext <32 x i16> %1 to <32 x i32> 500 %4 = zext <32 x i16> %2 to <32 x i32> 501 %5 = add nuw nsw <32 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 502 %6 = add nuw nsw <32 x i32> %5, %4 503 %7 = lshr <32 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 504 %8 = trunc <32 x i32> %7 to <32 x i16> 505 store <32 x i16> %8, ptr undef, align 4 506 ret void 507} 508 509define void @avg_v40i16(ptr %a, ptr %b) nounwind { 510; SSE2-LABEL: avg_v40i16: 511; SSE2: # %bb.0: 512; SSE2-NEXT: movdqa (%rsi), %xmm0 513; SSE2-NEXT: movdqa 16(%rsi), %xmm1 514; SSE2-NEXT: movdqa 32(%rsi), %xmm2 515; SSE2-NEXT: movdqa 48(%rsi), %xmm3 516; SSE2-NEXT: pavgw (%rdi), %xmm0 517; SSE2-NEXT: pavgw 16(%rdi), %xmm1 518; SSE2-NEXT: pavgw 32(%rdi), %xmm2 519; SSE2-NEXT: pavgw 48(%rdi), %xmm3 520; SSE2-NEXT: movdqa 64(%rsi), %xmm4 521; SSE2-NEXT: pavgw 64(%rdi), %xmm4 522; SSE2-NEXT: movdqu %xmm4, (%rax) 523; SSE2-NEXT: movdqu %xmm3, (%rax) 524; SSE2-NEXT: movdqu %xmm2, (%rax) 525; SSE2-NEXT: movdqu %xmm1, (%rax) 526; SSE2-NEXT: movdqu %xmm0, (%rax) 527; SSE2-NEXT: retq 528; 529; AVX1-LABEL: avg_v40i16: 530; AVX1: # %bb.0: 531; AVX1-NEXT: vmovdqa 64(%rsi), %xmm0 532; AVX1-NEXT: vpavgw 64(%rdi), %xmm0, %xmm0 533; AVX1-NEXT: vmovdqa (%rsi), %xmm1 534; AVX1-NEXT: vmovdqa 16(%rsi), %xmm2 535; AVX1-NEXT: vmovdqa 32(%rsi), %xmm3 536; AVX1-NEXT: vmovdqa 48(%rsi), %xmm4 537; AVX1-NEXT: vpavgw (%rdi), %xmm1, %xmm1 538; AVX1-NEXT: vpavgw 16(%rdi), %xmm2, %xmm2 539; AVX1-NEXT: vpavgw 32(%rdi), %xmm3, %xmm3 540; AVX1-NEXT: vpavgw 48(%rdi), %xmm4, %xmm4 541; AVX1-NEXT: vmovdqu %xmm4, (%rax) 542; AVX1-NEXT: vmovdqu %xmm3, (%rax) 543; AVX1-NEXT: vmovdqu %xmm2, (%rax) 544; AVX1-NEXT: vmovdqu %xmm1, (%rax) 545; AVX1-NEXT: vmovdqu %xmm0, (%rax) 546; AVX1-NEXT: retq 547; 548; AVX2-LABEL: avg_v40i16: 549; AVX2: # %bb.0: 550; AVX2-NEXT: vmovdqa (%rsi), %ymm0 551; AVX2-NEXT: vmovdqa 32(%rsi), %ymm1 552; AVX2-NEXT: vpavgw (%rdi), %ymm0, %ymm0 553; AVX2-NEXT: vpavgw 32(%rdi), %ymm1, %ymm1 554; AVX2-NEXT: vmovdqa 64(%rsi), %xmm2 555; AVX2-NEXT: vpavgw 64(%rdi), %xmm2, %xmm2 556; AVX2-NEXT: vmovdqu %xmm2, (%rax) 557; AVX2-NEXT: vmovdqu %ymm1, (%rax) 558; AVX2-NEXT: vmovdqu %ymm0, (%rax) 559; AVX2-NEXT: vzeroupper 560; AVX2-NEXT: retq 561; 562; AVX512F-LABEL: avg_v40i16: 563; AVX512F: # %bb.0: 564; AVX512F-NEXT: vmovdqa (%rsi), %ymm0 565; AVX512F-NEXT: vmovdqa 32(%rsi), %ymm1 566; AVX512F-NEXT: vpavgw (%rdi), %ymm0, %ymm0 567; AVX512F-NEXT: vpavgw 32(%rdi), %ymm1, %ymm1 568; AVX512F-NEXT: vmovdqa 64(%rsi), %xmm2 569; AVX512F-NEXT: vpavgw 64(%rdi), %xmm2, %xmm2 570; AVX512F-NEXT: vmovdqu %ymm1, (%rax) 571; AVX512F-NEXT: vmovdqu %ymm0, (%rax) 572; AVX512F-NEXT: vmovdqu %xmm2, (%rax) 573; AVX512F-NEXT: vzeroupper 574; AVX512F-NEXT: retq 575; 576; AVX512BW-LABEL: avg_v40i16: 577; AVX512BW: # %bb.0: 578; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm0 579; AVX512BW-NEXT: vpavgw (%rdi), %zmm0, %zmm0 580; AVX512BW-NEXT: vmovdqa 64(%rsi), %xmm1 581; AVX512BW-NEXT: vpavgw 64(%rdi), %xmm1, %xmm1 582; AVX512BW-NEXT: vmovdqu %xmm1, (%rax) 583; AVX512BW-NEXT: vmovdqu64 %zmm0, (%rax) 584; AVX512BW-NEXT: vzeroupper 585; AVX512BW-NEXT: retq 586 %1 = load <40 x i16>, ptr %a 587 %2 = load <40 x i16>, ptr %b 588 %3 = zext <40 x i16> %1 to <40 x i32> 589 %4 = zext <40 x i16> %2 to <40 x i32> 590 %5 = add nuw nsw <40 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 591 %6 = add nuw nsw <40 x i32> %5, %4 592 %7 = lshr <40 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 593 %8 = trunc <40 x i32> %7 to <40 x i16> 594 store <40 x i16> %8, ptr undef, align 4 595 ret void 596} 597 598define void @avg_v4i8_2(ptr %a, ptr %b) nounwind { 599; SSE2-LABEL: avg_v4i8_2: 600; SSE2: # %bb.0: 601; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero 602; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero 603; SSE2-NEXT: pavgb %xmm0, %xmm1 604; SSE2-NEXT: movd %xmm1, (%rax) 605; SSE2-NEXT: retq 606; 607; AVX-LABEL: avg_v4i8_2: 608; AVX: # %bb.0: 609; AVX-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero 610; AVX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero 611; AVX-NEXT: vpavgb %xmm1, %xmm0, %xmm0 612; AVX-NEXT: vmovd %xmm0, (%rax) 613; AVX-NEXT: retq 614 %1 = load <4 x i8>, ptr %a 615 %2 = load <4 x i8>, ptr %b 616 %3 = zext <4 x i8> %1 to <4 x i32> 617 %4 = zext <4 x i8> %2 to <4 x i32> 618 %5 = add nuw nsw <4 x i32> %3, %4 619 %6 = add nuw nsw <4 x i32> %5, <i32 1, i32 1, i32 1, i32 1> 620 %7 = lshr <4 x i32> %6, <i32 1, i32 1, i32 1, i32 1> 621 %8 = trunc <4 x i32> %7 to <4 x i8> 622 store <4 x i8> %8, ptr undef, align 4 623 ret void 624} 625 626define void @avg_v8i8_2(ptr %a, ptr %b) nounwind { 627; SSE2-LABEL: avg_v8i8_2: 628; SSE2: # %bb.0: 629; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero 630; SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero 631; SSE2-NEXT: pavgb %xmm0, %xmm1 632; SSE2-NEXT: movq %xmm1, (%rax) 633; SSE2-NEXT: retq 634; 635; AVX-LABEL: avg_v8i8_2: 636; AVX: # %bb.0: 637; AVX-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 638; AVX-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero 639; AVX-NEXT: vpavgb %xmm1, %xmm0, %xmm0 640; AVX-NEXT: vmovq %xmm0, (%rax) 641; AVX-NEXT: retq 642 %1 = load <8 x i8>, ptr %a 643 %2 = load <8 x i8>, ptr %b 644 %3 = zext <8 x i8> %1 to <8 x i32> 645 %4 = zext <8 x i8> %2 to <8 x i32> 646 %5 = add nuw nsw <8 x i32> %3, %4 647 %6 = add nuw nsw <8 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 648 %7 = lshr <8 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 649 %8 = trunc <8 x i32> %7 to <8 x i8> 650 store <8 x i8> %8, ptr undef, align 4 651 ret void 652} 653 654define void @avg_v16i8_2(ptr %a, ptr %b) nounwind { 655; SSE2-LABEL: avg_v16i8_2: 656; SSE2: # %bb.0: 657; SSE2-NEXT: movdqa (%rdi), %xmm0 658; SSE2-NEXT: pavgb (%rsi), %xmm0 659; SSE2-NEXT: movdqu %xmm0, (%rax) 660; SSE2-NEXT: retq 661; 662; AVX-LABEL: avg_v16i8_2: 663; AVX: # %bb.0: 664; AVX-NEXT: vmovdqa (%rdi), %xmm0 665; AVX-NEXT: vpavgb (%rsi), %xmm0, %xmm0 666; AVX-NEXT: vmovdqu %xmm0, (%rax) 667; AVX-NEXT: retq 668 %1 = load <16 x i8>, ptr %a 669 %2 = load <16 x i8>, ptr %b 670 %3 = zext <16 x i8> %1 to <16 x i32> 671 %4 = zext <16 x i8> %2 to <16 x i32> 672 %5 = add nuw nsw <16 x i32> %3, %4 673 %6 = add nuw nsw <16 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 674 %7 = lshr <16 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 675 %8 = trunc <16 x i32> %7 to <16 x i8> 676 store <16 x i8> %8, ptr undef, align 4 677 ret void 678} 679 680define void @avg_v32i8_2(ptr %a, ptr %b) nounwind { 681; SSE2-LABEL: avg_v32i8_2: 682; SSE2: # %bb.0: 683; SSE2-NEXT: movdqa (%rdi), %xmm0 684; SSE2-NEXT: movdqa 16(%rdi), %xmm1 685; SSE2-NEXT: pavgb (%rsi), %xmm0 686; SSE2-NEXT: pavgb 16(%rsi), %xmm1 687; SSE2-NEXT: movdqu %xmm1, (%rax) 688; SSE2-NEXT: movdqu %xmm0, (%rax) 689; SSE2-NEXT: retq 690; 691; AVX1-LABEL: avg_v32i8_2: 692; AVX1: # %bb.0: 693; AVX1-NEXT: vmovdqa (%rdi), %xmm0 694; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 695; AVX1-NEXT: vpavgb (%rsi), %xmm0, %xmm0 696; AVX1-NEXT: vpavgb 16(%rsi), %xmm1, %xmm1 697; AVX1-NEXT: vmovdqu %xmm1, (%rax) 698; AVX1-NEXT: vmovdqu %xmm0, (%rax) 699; AVX1-NEXT: retq 700; 701; AVX2-LABEL: avg_v32i8_2: 702; AVX2: # %bb.0: 703; AVX2-NEXT: vmovdqa (%rdi), %ymm0 704; AVX2-NEXT: vpavgb (%rsi), %ymm0, %ymm0 705; AVX2-NEXT: vmovdqu %ymm0, (%rax) 706; AVX2-NEXT: vzeroupper 707; AVX2-NEXT: retq 708; 709; AVX512-LABEL: avg_v32i8_2: 710; AVX512: # %bb.0: 711; AVX512-NEXT: vmovdqa (%rdi), %ymm0 712; AVX512-NEXT: vpavgb (%rsi), %ymm0, %ymm0 713; AVX512-NEXT: vmovdqu %ymm0, (%rax) 714; AVX512-NEXT: vzeroupper 715; AVX512-NEXT: retq 716 %1 = load <32 x i8>, ptr %a 717 %2 = load <32 x i8>, ptr %b 718 %3 = zext <32 x i8> %1 to <32 x i32> 719 %4 = zext <32 x i8> %2 to <32 x i32> 720 %5 = add nuw nsw <32 x i32> %3, %4 721 %6 = add nuw nsw <32 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 722 %7 = lshr <32 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 723 %8 = trunc <32 x i32> %7 to <32 x i8> 724 store <32 x i8> %8, ptr undef, align 4 725 ret void 726} 727 728define void @avg_v64i8_2(ptr %a, ptr %b) nounwind { 729; SSE2-LABEL: avg_v64i8_2: 730; SSE2: # %bb.0: 731; SSE2-NEXT: movdqa (%rsi), %xmm0 732; SSE2-NEXT: movdqa 16(%rsi), %xmm1 733; SSE2-NEXT: movdqa 32(%rsi), %xmm2 734; SSE2-NEXT: movdqa 48(%rsi), %xmm3 735; SSE2-NEXT: pavgb %xmm0, %xmm0 736; SSE2-NEXT: pavgb %xmm1, %xmm1 737; SSE2-NEXT: pavgb %xmm2, %xmm2 738; SSE2-NEXT: pavgb %xmm3, %xmm3 739; SSE2-NEXT: movdqu %xmm3, (%rax) 740; SSE2-NEXT: movdqu %xmm2, (%rax) 741; SSE2-NEXT: movdqu %xmm1, (%rax) 742; SSE2-NEXT: movdqu %xmm0, (%rax) 743; SSE2-NEXT: retq 744; 745; AVX1-LABEL: avg_v64i8_2: 746; AVX1: # %bb.0: 747; AVX1-NEXT: vmovdqa (%rsi), %xmm0 748; AVX1-NEXT: vmovdqa 16(%rsi), %xmm1 749; AVX1-NEXT: vmovdqa 32(%rsi), %xmm2 750; AVX1-NEXT: vmovdqa 48(%rsi), %xmm3 751; AVX1-NEXT: vpavgb %xmm0, %xmm0, %xmm0 752; AVX1-NEXT: vpavgb %xmm1, %xmm1, %xmm1 753; AVX1-NEXT: vpavgb %xmm2, %xmm2, %xmm2 754; AVX1-NEXT: vpavgb %xmm3, %xmm3, %xmm3 755; AVX1-NEXT: vmovdqu %xmm3, (%rax) 756; AVX1-NEXT: vmovdqu %xmm2, (%rax) 757; AVX1-NEXT: vmovdqu %xmm1, (%rax) 758; AVX1-NEXT: vmovdqu %xmm0, (%rax) 759; AVX1-NEXT: retq 760; 761; AVX2-LABEL: avg_v64i8_2: 762; AVX2: # %bb.0: 763; AVX2-NEXT: vmovdqa (%rsi), %ymm0 764; AVX2-NEXT: vmovdqa 32(%rsi), %ymm1 765; AVX2-NEXT: vpavgb %ymm0, %ymm0, %ymm0 766; AVX2-NEXT: vpavgb %ymm1, %ymm1, %ymm1 767; AVX2-NEXT: vmovdqu %ymm1, (%rax) 768; AVX2-NEXT: vmovdqu %ymm0, (%rax) 769; AVX2-NEXT: vzeroupper 770; AVX2-NEXT: retq 771; 772; AVX512F-LABEL: avg_v64i8_2: 773; AVX512F: # %bb.0: 774; AVX512F-NEXT: vmovdqa (%rsi), %ymm0 775; AVX512F-NEXT: vmovdqa 32(%rsi), %ymm1 776; AVX512F-NEXT: vpavgb %ymm0, %ymm0, %ymm0 777; AVX512F-NEXT: vpavgb %ymm1, %ymm1, %ymm1 778; AVX512F-NEXT: vmovdqu %ymm1, (%rax) 779; AVX512F-NEXT: vmovdqu %ymm0, (%rax) 780; AVX512F-NEXT: vzeroupper 781; AVX512F-NEXT: retq 782; 783; AVX512BW-LABEL: avg_v64i8_2: 784; AVX512BW: # %bb.0: 785; AVX512BW-NEXT: vmovdqa64 (%rsi), %zmm0 786; AVX512BW-NEXT: vpavgb %zmm0, %zmm0, %zmm0 787; AVX512BW-NEXT: vmovdqu64 %zmm0, (%rax) 788; AVX512BW-NEXT: vzeroupper 789; AVX512BW-NEXT: retq 790 %1 = load <64 x i8>, ptr %a 791 %2 = load <64 x i8>, ptr %b 792 %3 = zext <64 x i8> %1 to <64 x i32> 793 %4 = zext <64 x i8> %2 to <64 x i32> 794 %5 = add nuw nsw <64 x i32> %4, %4 795 %6 = add nuw nsw <64 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 796 %7 = lshr <64 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 797 %8 = trunc <64 x i32> %7 to <64 x i8> 798 store <64 x i8> %8, ptr undef, align 4 799 ret void 800} 801 802 803define void @avg_v4i16_2(ptr %a, ptr %b) nounwind { 804; SSE2-LABEL: avg_v4i16_2: 805; SSE2: # %bb.0: 806; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero 807; SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero 808; SSE2-NEXT: pavgw %xmm0, %xmm1 809; SSE2-NEXT: movq %xmm1, (%rax) 810; SSE2-NEXT: retq 811; 812; AVX-LABEL: avg_v4i16_2: 813; AVX: # %bb.0: 814; AVX-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 815; AVX-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero 816; AVX-NEXT: vpavgw %xmm1, %xmm0, %xmm0 817; AVX-NEXT: vmovq %xmm0, (%rax) 818; AVX-NEXT: retq 819 %1 = load <4 x i16>, ptr %a 820 %2 = load <4 x i16>, ptr %b 821 %3 = zext <4 x i16> %1 to <4 x i32> 822 %4 = zext <4 x i16> %2 to <4 x i32> 823 %5 = add nuw nsw <4 x i32> %3, %4 824 %6 = add nuw nsw <4 x i32> %5, <i32 1, i32 1, i32 1, i32 1> 825 %7 = lshr <4 x i32> %6, <i32 1, i32 1, i32 1, i32 1> 826 %8 = trunc <4 x i32> %7 to <4 x i16> 827 store <4 x i16> %8, ptr undef, align 4 828 ret void 829} 830 831define void @avg_v8i16_2(ptr %a, ptr %b) nounwind { 832; SSE2-LABEL: avg_v8i16_2: 833; SSE2: # %bb.0: 834; SSE2-NEXT: movdqa (%rdi), %xmm0 835; SSE2-NEXT: pavgw (%rsi), %xmm0 836; SSE2-NEXT: movdqu %xmm0, (%rax) 837; SSE2-NEXT: retq 838; 839; AVX-LABEL: avg_v8i16_2: 840; AVX: # %bb.0: 841; AVX-NEXT: vmovdqa (%rdi), %xmm0 842; AVX-NEXT: vpavgw (%rsi), %xmm0, %xmm0 843; AVX-NEXT: vmovdqu %xmm0, (%rax) 844; AVX-NEXT: retq 845 %1 = load <8 x i16>, ptr %a 846 %2 = load <8 x i16>, ptr %b 847 %3 = zext <8 x i16> %1 to <8 x i32> 848 %4 = zext <8 x i16> %2 to <8 x i32> 849 %5 = add nuw nsw <8 x i32> %3, %4 850 %6 = add nuw nsw <8 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 851 %7 = lshr <8 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 852 %8 = trunc <8 x i32> %7 to <8 x i16> 853 store <8 x i16> %8, ptr undef, align 4 854 ret void 855} 856 857define void @avg_v16i16_2(ptr %a, ptr %b) nounwind { 858; SSE2-LABEL: avg_v16i16_2: 859; SSE2: # %bb.0: 860; SSE2-NEXT: movdqa (%rdi), %xmm0 861; SSE2-NEXT: movdqa 16(%rdi), %xmm1 862; SSE2-NEXT: pavgw (%rsi), %xmm0 863; SSE2-NEXT: pavgw 16(%rsi), %xmm1 864; SSE2-NEXT: movdqu %xmm1, (%rax) 865; SSE2-NEXT: movdqu %xmm0, (%rax) 866; SSE2-NEXT: retq 867; 868; AVX1-LABEL: avg_v16i16_2: 869; AVX1: # %bb.0: 870; AVX1-NEXT: vmovdqa (%rdi), %xmm0 871; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 872; AVX1-NEXT: vpavgw (%rsi), %xmm0, %xmm0 873; AVX1-NEXT: vpavgw 16(%rsi), %xmm1, %xmm1 874; AVX1-NEXT: vmovdqu %xmm1, (%rax) 875; AVX1-NEXT: vmovdqu %xmm0, (%rax) 876; AVX1-NEXT: retq 877; 878; AVX2-LABEL: avg_v16i16_2: 879; AVX2: # %bb.0: 880; AVX2-NEXT: vmovdqa (%rdi), %ymm0 881; AVX2-NEXT: vpavgw (%rsi), %ymm0, %ymm0 882; AVX2-NEXT: vmovdqu %ymm0, (%rax) 883; AVX2-NEXT: vzeroupper 884; AVX2-NEXT: retq 885; 886; AVX512-LABEL: avg_v16i16_2: 887; AVX512: # %bb.0: 888; AVX512-NEXT: vmovdqa (%rdi), %ymm0 889; AVX512-NEXT: vpavgw (%rsi), %ymm0, %ymm0 890; AVX512-NEXT: vmovdqu %ymm0, (%rax) 891; AVX512-NEXT: vzeroupper 892; AVX512-NEXT: retq 893 %1 = load <16 x i16>, ptr %a 894 %2 = load <16 x i16>, ptr %b 895 %3 = zext <16 x i16> %1 to <16 x i32> 896 %4 = zext <16 x i16> %2 to <16 x i32> 897 %5 = add nuw nsw <16 x i32> %3, %4 898 %6 = add nuw nsw <16 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 899 %7 = lshr <16 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 900 %8 = trunc <16 x i32> %7 to <16 x i16> 901 store <16 x i16> %8, ptr undef, align 4 902 ret void 903} 904 905define void @avg_v32i16_2(ptr %a, ptr %b) nounwind { 906; SSE2-LABEL: avg_v32i16_2: 907; SSE2: # %bb.0: 908; SSE2-NEXT: movdqa (%rdi), %xmm0 909; SSE2-NEXT: movdqa 16(%rdi), %xmm1 910; SSE2-NEXT: movdqa 32(%rdi), %xmm2 911; SSE2-NEXT: movdqa 48(%rdi), %xmm3 912; SSE2-NEXT: pavgw (%rsi), %xmm0 913; SSE2-NEXT: pavgw 16(%rsi), %xmm1 914; SSE2-NEXT: pavgw 32(%rsi), %xmm2 915; SSE2-NEXT: pavgw 48(%rsi), %xmm3 916; SSE2-NEXT: movdqu %xmm3, (%rax) 917; SSE2-NEXT: movdqu %xmm2, (%rax) 918; SSE2-NEXT: movdqu %xmm1, (%rax) 919; SSE2-NEXT: movdqu %xmm0, (%rax) 920; SSE2-NEXT: retq 921; 922; AVX1-LABEL: avg_v32i16_2: 923; AVX1: # %bb.0: 924; AVX1-NEXT: vmovdqa (%rdi), %xmm0 925; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 926; AVX1-NEXT: vmovdqa 32(%rdi), %xmm2 927; AVX1-NEXT: vmovdqa 48(%rdi), %xmm3 928; AVX1-NEXT: vpavgw (%rsi), %xmm0, %xmm0 929; AVX1-NEXT: vpavgw 16(%rsi), %xmm1, %xmm1 930; AVX1-NEXT: vpavgw 32(%rsi), %xmm2, %xmm2 931; AVX1-NEXT: vpavgw 48(%rsi), %xmm3, %xmm3 932; AVX1-NEXT: vmovdqu %xmm3, (%rax) 933; AVX1-NEXT: vmovdqu %xmm2, (%rax) 934; AVX1-NEXT: vmovdqu %xmm1, (%rax) 935; AVX1-NEXT: vmovdqu %xmm0, (%rax) 936; AVX1-NEXT: retq 937; 938; AVX2-LABEL: avg_v32i16_2: 939; AVX2: # %bb.0: 940; AVX2-NEXT: vmovdqa (%rdi), %ymm0 941; AVX2-NEXT: vmovdqa 32(%rdi), %ymm1 942; AVX2-NEXT: vpavgw (%rsi), %ymm0, %ymm0 943; AVX2-NEXT: vpavgw 32(%rsi), %ymm1, %ymm1 944; AVX2-NEXT: vmovdqu %ymm1, (%rax) 945; AVX2-NEXT: vmovdqu %ymm0, (%rax) 946; AVX2-NEXT: vzeroupper 947; AVX2-NEXT: retq 948; 949; AVX512F-LABEL: avg_v32i16_2: 950; AVX512F: # %bb.0: 951; AVX512F-NEXT: vmovdqa (%rdi), %ymm0 952; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm1 953; AVX512F-NEXT: vpavgw (%rsi), %ymm0, %ymm0 954; AVX512F-NEXT: vpavgw 32(%rsi), %ymm1, %ymm1 955; AVX512F-NEXT: vmovdqu %ymm1, (%rax) 956; AVX512F-NEXT: vmovdqu %ymm0, (%rax) 957; AVX512F-NEXT: vzeroupper 958; AVX512F-NEXT: retq 959; 960; AVX512BW-LABEL: avg_v32i16_2: 961; AVX512BW: # %bb.0: 962; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0 963; AVX512BW-NEXT: vpavgw (%rsi), %zmm0, %zmm0 964; AVX512BW-NEXT: vmovdqu64 %zmm0, (%rax) 965; AVX512BW-NEXT: vzeroupper 966; AVX512BW-NEXT: retq 967 %1 = load <32 x i16>, ptr %a 968 %2 = load <32 x i16>, ptr %b 969 %3 = zext <32 x i16> %1 to <32 x i32> 970 %4 = zext <32 x i16> %2 to <32 x i32> 971 %5 = add nuw nsw <32 x i32> %3, %4 972 %6 = add nuw nsw <32 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 973 %7 = lshr <32 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 974 %8 = trunc <32 x i32> %7 to <32 x i16> 975 store <32 x i16> %8, ptr undef, align 4 976 ret void 977} 978 979define void @avg_v4i8_const(ptr %a) nounwind { 980; SSE2-LABEL: avg_v4i8_const: 981; SSE2: # %bb.0: 982; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero 983; SSE2-NEXT: pavgb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 984; SSE2-NEXT: movd %xmm0, (%rax) 985; SSE2-NEXT: retq 986; 987; AVX-LABEL: avg_v4i8_const: 988; AVX: # %bb.0: 989; AVX-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero 990; AVX-NEXT: vpavgb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 991; AVX-NEXT: vmovd %xmm0, (%rax) 992; AVX-NEXT: retq 993 %1 = load <4 x i8>, ptr %a 994 %2 = zext <4 x i8> %1 to <4 x i32> 995 %3 = add nuw nsw <4 x i32> %2, <i32 1, i32 2, i32 3, i32 4> 996 %4 = lshr <4 x i32> %3, <i32 1, i32 1, i32 1, i32 1> 997 %5 = trunc <4 x i32> %4 to <4 x i8> 998 store <4 x i8> %5, ptr undef, align 4 999 ret void 1000} 1001 1002define void @avg_v8i8_const(ptr %a) nounwind { 1003; SSE2-LABEL: avg_v8i8_const: 1004; SSE2: # %bb.0: 1005; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero 1006; SSE2-NEXT: pavgb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 1007; SSE2-NEXT: movq %xmm0, (%rax) 1008; SSE2-NEXT: retq 1009; 1010; AVX-LABEL: avg_v8i8_const: 1011; AVX: # %bb.0: 1012; AVX-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 1013; AVX-NEXT: vpavgb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 1014; AVX-NEXT: vmovq %xmm0, (%rax) 1015; AVX-NEXT: retq 1016 %1 = load <8 x i8>, ptr %a 1017 %2 = zext <8 x i8> %1 to <8 x i32> 1018 %3 = add nuw nsw <8 x i32> %2, <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8> 1019 %4 = lshr <8 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1020 %5 = trunc <8 x i32> %4 to <8 x i8> 1021 store <8 x i8> %5, ptr undef, align 4 1022 ret void 1023} 1024 1025define void @avg_v16i8_const(ptr %a) nounwind { 1026; SSE2-LABEL: avg_v16i8_const: 1027; SSE2: # %bb.0: 1028; SSE2-NEXT: movdqa (%rdi), %xmm0 1029; SSE2-NEXT: pavgb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 1030; SSE2-NEXT: movdqu %xmm0, (%rax) 1031; SSE2-NEXT: retq 1032; 1033; AVX-LABEL: avg_v16i8_const: 1034; AVX: # %bb.0: 1035; AVX-NEXT: vmovdqa (%rdi), %xmm0 1036; AVX-NEXT: vpavgb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 1037; AVX-NEXT: vmovdqu %xmm0, (%rax) 1038; AVX-NEXT: retq 1039 %1 = load <16 x i8>, ptr %a 1040 %2 = zext <16 x i8> %1 to <16 x i32> 1041 %3 = add nuw nsw <16 x i32> %2, <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8> 1042 %4 = lshr <16 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1043 %5 = trunc <16 x i32> %4 to <16 x i8> 1044 store <16 x i8> %5, ptr undef, align 4 1045 ret void 1046} 1047 1048define void @avg_v32i8_const(ptr %a) nounwind { 1049; SSE2-LABEL: avg_v32i8_const: 1050; SSE2: # %bb.0: 1051; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7] 1052; SSE2-NEXT: movdqa (%rdi), %xmm1 1053; SSE2-NEXT: pavgb %xmm0, %xmm1 1054; SSE2-NEXT: pavgb 16(%rdi), %xmm0 1055; SSE2-NEXT: movdqu %xmm0, (%rax) 1056; SSE2-NEXT: movdqu %xmm1, (%rax) 1057; SSE2-NEXT: retq 1058; 1059; AVX1-LABEL: avg_v32i8_const: 1060; AVX1: # %bb.0: 1061; AVX1-NEXT: vmovddup {{.*#+}} xmm0 = [506097522914230528,506097522914230528] 1062; AVX1-NEXT: # xmm0 = mem[0,0] 1063; AVX1-NEXT: vpavgb (%rdi), %xmm0, %xmm1 1064; AVX1-NEXT: vpavgb 16(%rdi), %xmm0, %xmm0 1065; AVX1-NEXT: vmovdqu %xmm0, (%rax) 1066; AVX1-NEXT: vmovdqu %xmm1, (%rax) 1067; AVX1-NEXT: retq 1068; 1069; AVX2-LABEL: avg_v32i8_const: 1070; AVX2: # %bb.0: 1071; AVX2-NEXT: vmovdqa (%rdi), %ymm0 1072; AVX2-NEXT: vpavgb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 1073; AVX2-NEXT: vmovdqu %ymm0, (%rax) 1074; AVX2-NEXT: vzeroupper 1075; AVX2-NEXT: retq 1076; 1077; AVX512-LABEL: avg_v32i8_const: 1078; AVX512: # %bb.0: 1079; AVX512-NEXT: vmovdqa (%rdi), %ymm0 1080; AVX512-NEXT: vpavgb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 1081; AVX512-NEXT: vmovdqu %ymm0, (%rax) 1082; AVX512-NEXT: vzeroupper 1083; AVX512-NEXT: retq 1084 %1 = load <32 x i8>, ptr %a 1085 %2 = zext <32 x i8> %1 to <32 x i32> 1086 %3 = add nuw nsw <32 x i32> %2, <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8> 1087 %4 = lshr <32 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1088 %5 = trunc <32 x i32> %4 to <32 x i8> 1089 store <32 x i8> %5, ptr undef, align 4 1090 ret void 1091} 1092 1093define void @avg_v64i8_const(ptr %a) nounwind { 1094; SSE2-LABEL: avg_v64i8_const: 1095; SSE2: # %bb.0: 1096; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7] 1097; SSE2-NEXT: movdqa (%rdi), %xmm1 1098; SSE2-NEXT: pavgb %xmm0, %xmm1 1099; SSE2-NEXT: movdqa 16(%rdi), %xmm2 1100; SSE2-NEXT: pavgb %xmm0, %xmm2 1101; SSE2-NEXT: movdqa 32(%rdi), %xmm3 1102; SSE2-NEXT: pavgb %xmm0, %xmm3 1103; SSE2-NEXT: pavgb 48(%rdi), %xmm0 1104; SSE2-NEXT: movdqu %xmm0, (%rax) 1105; SSE2-NEXT: movdqu %xmm3, (%rax) 1106; SSE2-NEXT: movdqu %xmm2, (%rax) 1107; SSE2-NEXT: movdqu %xmm1, (%rax) 1108; SSE2-NEXT: retq 1109; 1110; AVX1-LABEL: avg_v64i8_const: 1111; AVX1: # %bb.0: 1112; AVX1-NEXT: vmovddup {{.*#+}} xmm0 = [506097522914230528,506097522914230528] 1113; AVX1-NEXT: # xmm0 = mem[0,0] 1114; AVX1-NEXT: vpavgb (%rdi), %xmm0, %xmm1 1115; AVX1-NEXT: vpavgb 16(%rdi), %xmm0, %xmm2 1116; AVX1-NEXT: vpavgb 32(%rdi), %xmm0, %xmm3 1117; AVX1-NEXT: vpavgb 48(%rdi), %xmm0, %xmm0 1118; AVX1-NEXT: vmovdqu %xmm0, (%rax) 1119; AVX1-NEXT: vmovdqu %xmm3, (%rax) 1120; AVX1-NEXT: vmovdqu %xmm2, (%rax) 1121; AVX1-NEXT: vmovdqu %xmm1, (%rax) 1122; AVX1-NEXT: retq 1123; 1124; AVX2-LABEL: avg_v64i8_const: 1125; AVX2: # %bb.0: 1126; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm0 = [506097522914230528,506097522914230528,506097522914230528,506097522914230528] 1127; AVX2-NEXT: vpavgb (%rdi), %ymm0, %ymm1 1128; AVX2-NEXT: vpavgb 32(%rdi), %ymm0, %ymm0 1129; AVX2-NEXT: vmovdqu %ymm0, (%rax) 1130; AVX2-NEXT: vmovdqu %ymm1, (%rax) 1131; AVX2-NEXT: vzeroupper 1132; AVX2-NEXT: retq 1133; 1134; AVX512F-LABEL: avg_v64i8_const: 1135; AVX512F: # %bb.0: 1136; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm0 = [506097522914230528,506097522914230528,506097522914230528,506097522914230528] 1137; AVX512F-NEXT: vpavgb (%rdi), %ymm0, %ymm1 1138; AVX512F-NEXT: vpavgb 32(%rdi), %ymm0, %ymm0 1139; AVX512F-NEXT: vmovdqu %ymm0, (%rax) 1140; AVX512F-NEXT: vmovdqu %ymm1, (%rax) 1141; AVX512F-NEXT: vzeroupper 1142; AVX512F-NEXT: retq 1143; 1144; AVX512BW-LABEL: avg_v64i8_const: 1145; AVX512BW: # %bb.0: 1146; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0 1147; AVX512BW-NEXT: vpavgb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 1148; AVX512BW-NEXT: vmovdqu64 %zmm0, (%rax) 1149; AVX512BW-NEXT: vzeroupper 1150; AVX512BW-NEXT: retq 1151 %1 = load <64 x i8>, ptr %a 1152 %2 = zext <64 x i8> %1 to <64 x i32> 1153 %3 = add nuw nsw <64 x i32> %2, <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8> 1154 %4 = lshr <64 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1155 %5 = trunc <64 x i32> %4 to <64 x i8> 1156 store <64 x i8> %5, ptr undef, align 4 1157 ret void 1158} 1159 1160define void @avg_v4i16_const(ptr %a) nounwind { 1161; SSE2-LABEL: avg_v4i16_const: 1162; SSE2: # %bb.0: 1163; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero 1164; SSE2-NEXT: pavgw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 1165; SSE2-NEXT: movq %xmm0, (%rax) 1166; SSE2-NEXT: retq 1167; 1168; AVX-LABEL: avg_v4i16_const: 1169; AVX: # %bb.0: 1170; AVX-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero 1171; AVX-NEXT: vpavgw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 1172; AVX-NEXT: vmovq %xmm0, (%rax) 1173; AVX-NEXT: retq 1174 %1 = load <4 x i16>, ptr %a 1175 %2 = zext <4 x i16> %1 to <4 x i32> 1176 %3 = add nuw nsw <4 x i32> %2, <i32 1, i32 2, i32 3, i32 4> 1177 %4 = lshr <4 x i32> %3, <i32 1, i32 1, i32 1, i32 1> 1178 %5 = trunc <4 x i32> %4 to <4 x i16> 1179 store <4 x i16> %5, ptr undef, align 4 1180 ret void 1181} 1182 1183define void @avg_v8i16_const(ptr %a) nounwind { 1184; SSE2-LABEL: avg_v8i16_const: 1185; SSE2: # %bb.0: 1186; SSE2-NEXT: movdqa (%rdi), %xmm0 1187; SSE2-NEXT: pavgw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 1188; SSE2-NEXT: movdqu %xmm0, (%rax) 1189; SSE2-NEXT: retq 1190; 1191; AVX-LABEL: avg_v8i16_const: 1192; AVX: # %bb.0: 1193; AVX-NEXT: vmovdqa (%rdi), %xmm0 1194; AVX-NEXT: vpavgw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 1195; AVX-NEXT: vmovdqu %xmm0, (%rax) 1196; AVX-NEXT: retq 1197 %1 = load <8 x i16>, ptr %a 1198 %2 = zext <8 x i16> %1 to <8 x i32> 1199 %3 = add nuw nsw <8 x i32> %2, <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8> 1200 %4 = lshr <8 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1201 %5 = trunc <8 x i32> %4 to <8 x i16> 1202 store <8 x i16> %5, ptr undef, align 4 1203 ret void 1204} 1205 1206define void @avg_v16i16_const(ptr %a) nounwind { 1207; SSE2-LABEL: avg_v16i16_const: 1208; SSE2: # %bb.0: 1209; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [0,1,2,3,4,5,6,7] 1210; SSE2-NEXT: movdqa (%rdi), %xmm1 1211; SSE2-NEXT: pavgw %xmm0, %xmm1 1212; SSE2-NEXT: pavgw 16(%rdi), %xmm0 1213; SSE2-NEXT: movdqu %xmm0, (%rax) 1214; SSE2-NEXT: movdqu %xmm1, (%rax) 1215; SSE2-NEXT: retq 1216; 1217; AVX1-LABEL: avg_v16i16_const: 1218; AVX1: # %bb.0: 1219; AVX1-NEXT: vmovdqa {{.*#+}} xmm0 = [0,1,2,3,4,5,6,7] 1220; AVX1-NEXT: vpavgw (%rdi), %xmm0, %xmm1 1221; AVX1-NEXT: vpavgw 16(%rdi), %xmm0, %xmm0 1222; AVX1-NEXT: vmovdqu %xmm0, (%rax) 1223; AVX1-NEXT: vmovdqu %xmm1, (%rax) 1224; AVX1-NEXT: retq 1225; 1226; AVX2-LABEL: avg_v16i16_const: 1227; AVX2: # %bb.0: 1228; AVX2-NEXT: vmovdqa (%rdi), %ymm0 1229; AVX2-NEXT: vpavgw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 1230; AVX2-NEXT: vmovdqu %ymm0, (%rax) 1231; AVX2-NEXT: vzeroupper 1232; AVX2-NEXT: retq 1233; 1234; AVX512-LABEL: avg_v16i16_const: 1235; AVX512: # %bb.0: 1236; AVX512-NEXT: vmovdqa (%rdi), %ymm0 1237; AVX512-NEXT: vpavgw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 1238; AVX512-NEXT: vmovdqu %ymm0, (%rax) 1239; AVX512-NEXT: vzeroupper 1240; AVX512-NEXT: retq 1241 %1 = load <16 x i16>, ptr %a 1242 %2 = zext <16 x i16> %1 to <16 x i32> 1243 %3 = add nuw nsw <16 x i32> %2, <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8> 1244 %4 = lshr <16 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1245 %5 = trunc <16 x i32> %4 to <16 x i16> 1246 store <16 x i16> %5, ptr undef, align 4 1247 ret void 1248} 1249 1250define void @avg_v32i16_const(ptr %a) nounwind { 1251; SSE2-LABEL: avg_v32i16_const: 1252; SSE2: # %bb.0: 1253; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [0,1,2,3,4,5,6,7] 1254; SSE2-NEXT: movdqa (%rdi), %xmm1 1255; SSE2-NEXT: pavgw %xmm0, %xmm1 1256; SSE2-NEXT: movdqa 16(%rdi), %xmm2 1257; SSE2-NEXT: pavgw %xmm0, %xmm2 1258; SSE2-NEXT: movdqa 32(%rdi), %xmm3 1259; SSE2-NEXT: pavgw %xmm0, %xmm3 1260; SSE2-NEXT: pavgw 48(%rdi), %xmm0 1261; SSE2-NEXT: movdqu %xmm0, (%rax) 1262; SSE2-NEXT: movdqu %xmm3, (%rax) 1263; SSE2-NEXT: movdqu %xmm2, (%rax) 1264; SSE2-NEXT: movdqu %xmm1, (%rax) 1265; SSE2-NEXT: retq 1266; 1267; AVX1-LABEL: avg_v32i16_const: 1268; AVX1: # %bb.0: 1269; AVX1-NEXT: vmovdqa {{.*#+}} xmm0 = [0,1,2,3,4,5,6,7] 1270; AVX1-NEXT: vpavgw (%rdi), %xmm0, %xmm1 1271; AVX1-NEXT: vpavgw 16(%rdi), %xmm0, %xmm2 1272; AVX1-NEXT: vpavgw 32(%rdi), %xmm0, %xmm3 1273; AVX1-NEXT: vpavgw 48(%rdi), %xmm0, %xmm0 1274; AVX1-NEXT: vmovdqu %xmm0, (%rax) 1275; AVX1-NEXT: vmovdqu %xmm3, (%rax) 1276; AVX1-NEXT: vmovdqu %xmm2, (%rax) 1277; AVX1-NEXT: vmovdqu %xmm1, (%rax) 1278; AVX1-NEXT: retq 1279; 1280; AVX2-LABEL: avg_v32i16_const: 1281; AVX2: # %bb.0: 1282; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7] 1283; AVX2-NEXT: # ymm0 = mem[0,1,0,1] 1284; AVX2-NEXT: vpavgw (%rdi), %ymm0, %ymm1 1285; AVX2-NEXT: vpavgw 32(%rdi), %ymm0, %ymm0 1286; AVX2-NEXT: vmovdqu %ymm0, (%rax) 1287; AVX2-NEXT: vmovdqu %ymm1, (%rax) 1288; AVX2-NEXT: vzeroupper 1289; AVX2-NEXT: retq 1290; 1291; AVX512F-LABEL: avg_v32i16_const: 1292; AVX512F: # %bb.0: 1293; AVX512F-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7] 1294; AVX512F-NEXT: # ymm0 = mem[0,1,0,1] 1295; AVX512F-NEXT: vpavgw (%rdi), %ymm0, %ymm1 1296; AVX512F-NEXT: vpavgw 32(%rdi), %ymm0, %ymm0 1297; AVX512F-NEXT: vmovdqu %ymm0, (%rax) 1298; AVX512F-NEXT: vmovdqu %ymm1, (%rax) 1299; AVX512F-NEXT: vzeroupper 1300; AVX512F-NEXT: retq 1301; 1302; AVX512BW-LABEL: avg_v32i16_const: 1303; AVX512BW: # %bb.0: 1304; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0 1305; AVX512BW-NEXT: vpavgw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 1306; AVX512BW-NEXT: vmovdqu64 %zmm0, (%rax) 1307; AVX512BW-NEXT: vzeroupper 1308; AVX512BW-NEXT: retq 1309 %1 = load <32 x i16>, ptr %a 1310 %2 = zext <32 x i16> %1 to <32 x i32> 1311 %3 = add nuw nsw <32 x i32> %2, <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8> 1312 %4 = lshr <32 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 1313 %5 = trunc <32 x i32> %4 to <32 x i16> 1314 store <32 x i16> %5, ptr undef, align 4 1315 ret void 1316} 1317 1318define <16 x i8> @avg_v16i8_3(<16 x i8> %a, <16 x i8> %b) nounwind { 1319; SSE2-LABEL: avg_v16i8_3: 1320; SSE2: # %bb.0: 1321; SSE2-NEXT: pavgb %xmm1, %xmm0 1322; SSE2-NEXT: retq 1323; 1324; AVX-LABEL: avg_v16i8_3: 1325; AVX: # %bb.0: 1326; AVX-NEXT: vpavgb %xmm1, %xmm0, %xmm0 1327; AVX-NEXT: retq 1328 %za = zext <16 x i8> %a to <16 x i16> 1329 %zb = zext <16 x i8> %b to <16 x i16> 1330 %add = add nuw nsw <16 x i16> %za, %zb 1331 %add1 = add nuw nsw <16 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> 1332 %lshr = lshr <16 x i16> %add1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> 1333 %res = trunc <16 x i16> %lshr to <16 x i8> 1334 ret <16 x i8> %res 1335} 1336 1337define <32 x i8> @avg_v32i8_3(<32 x i8> %a, <32 x i8> %b) nounwind { 1338; SSE2-LABEL: avg_v32i8_3: 1339; SSE2: # %bb.0: 1340; SSE2-NEXT: pavgb %xmm2, %xmm0 1341; SSE2-NEXT: pavgb %xmm3, %xmm1 1342; SSE2-NEXT: retq 1343; 1344; AVX1-LABEL: avg_v32i8_3: 1345; AVX1: # %bb.0: 1346; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 1347; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 1348; AVX1-NEXT: vpavgb %xmm2, %xmm3, %xmm2 1349; AVX1-NEXT: vpavgb %xmm1, %xmm0, %xmm0 1350; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 1351; AVX1-NEXT: retq 1352; 1353; AVX2-LABEL: avg_v32i8_3: 1354; AVX2: # %bb.0: 1355; AVX2-NEXT: vpavgb %ymm1, %ymm0, %ymm0 1356; AVX2-NEXT: retq 1357; 1358; AVX512-LABEL: avg_v32i8_3: 1359; AVX512: # %bb.0: 1360; AVX512-NEXT: vpavgb %ymm1, %ymm0, %ymm0 1361; AVX512-NEXT: retq 1362 %za = zext <32 x i8> %a to <32 x i16> 1363 %zb = zext <32 x i8> %b to <32 x i16> 1364 %add = add nuw nsw <32 x i16> %za, %zb 1365 %add1 = add nuw nsw <32 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> 1366 %lshr = lshr <32 x i16> %add1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> 1367 %res = trunc <32 x i16> %lshr to <32 x i8> 1368 ret <32 x i8> %res 1369} 1370 1371define <64 x i8> @avg_v64i8_3(<64 x i8> %a, <64 x i8> %b) nounwind { 1372; SSE2-LABEL: avg_v64i8_3: 1373; SSE2: # %bb.0: 1374; SSE2-NEXT: pavgb %xmm4, %xmm0 1375; SSE2-NEXT: pavgb %xmm5, %xmm1 1376; SSE2-NEXT: pavgb %xmm6, %xmm2 1377; SSE2-NEXT: pavgb %xmm7, %xmm3 1378; SSE2-NEXT: retq 1379; 1380; AVX1-LABEL: avg_v64i8_3: 1381; AVX1: # %bb.0: 1382; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4 1383; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5 1384; AVX1-NEXT: vpavgb %xmm4, %xmm5, %xmm4 1385; AVX1-NEXT: vpavgb %xmm2, %xmm0, %xmm0 1386; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0 1387; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2 1388; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4 1389; AVX1-NEXT: vpavgb %xmm2, %xmm4, %xmm2 1390; AVX1-NEXT: vpavgb %xmm3, %xmm1, %xmm1 1391; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 1392; AVX1-NEXT: retq 1393; 1394; AVX2-LABEL: avg_v64i8_3: 1395; AVX2: # %bb.0: 1396; AVX2-NEXT: vpavgb %ymm2, %ymm0, %ymm0 1397; AVX2-NEXT: vpavgb %ymm3, %ymm1, %ymm1 1398; AVX2-NEXT: retq 1399; 1400; AVX512F-LABEL: avg_v64i8_3: 1401; AVX512F: # %bb.0: 1402; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm2 1403; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm3 1404; AVX512F-NEXT: vpavgb %ymm2, %ymm3, %ymm2 1405; AVX512F-NEXT: vpavgb %ymm1, %ymm0, %ymm0 1406; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 1407; AVX512F-NEXT: retq 1408; 1409; AVX512BW-LABEL: avg_v64i8_3: 1410; AVX512BW: # %bb.0: 1411; AVX512BW-NEXT: vpavgb %zmm1, %zmm0, %zmm0 1412; AVX512BW-NEXT: retq 1413 %za = zext <64 x i8> %a to <64 x i16> 1414 %zb = zext <64 x i8> %b to <64 x i16> 1415 %add = add nuw nsw <64 x i16> %za, %zb 1416 %add1 = add nuw nsw <64 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> 1417 %lshr = lshr <64 x i16> %add1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> 1418 %res = trunc <64 x i16> %lshr to <64 x i8> 1419 ret <64 x i8> %res 1420} 1421 1422define <512 x i8> @avg_v512i8_3(<512 x i8> %a, <512 x i8> %b) nounwind { 1423; SSE2-LABEL: avg_v512i8_3: 1424; SSE2: # %bb.0: 1425; SSE2-NEXT: movq %rdi, %rax 1426; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1427; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1428; SSE2-NEXT: movdqa %xmm8, 496(%rdi) 1429; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1430; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1431; SSE2-NEXT: movdqa %xmm8, 480(%rdi) 1432; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1433; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1434; SSE2-NEXT: movdqa %xmm8, 464(%rdi) 1435; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1436; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1437; SSE2-NEXT: movdqa %xmm8, 448(%rdi) 1438; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1439; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1440; SSE2-NEXT: movdqa %xmm8, 432(%rdi) 1441; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1442; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1443; SSE2-NEXT: movdqa %xmm8, 416(%rdi) 1444; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1445; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1446; SSE2-NEXT: movdqa %xmm8, 400(%rdi) 1447; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1448; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1449; SSE2-NEXT: movdqa %xmm8, 384(%rdi) 1450; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1451; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1452; SSE2-NEXT: movdqa %xmm8, 368(%rdi) 1453; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1454; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1455; SSE2-NEXT: movdqa %xmm8, 352(%rdi) 1456; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1457; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1458; SSE2-NEXT: movdqa %xmm8, 336(%rdi) 1459; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1460; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1461; SSE2-NEXT: movdqa %xmm8, 320(%rdi) 1462; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1463; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1464; SSE2-NEXT: movdqa %xmm8, 304(%rdi) 1465; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1466; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1467; SSE2-NEXT: movdqa %xmm8, 288(%rdi) 1468; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1469; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1470; SSE2-NEXT: movdqa %xmm8, 272(%rdi) 1471; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1472; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1473; SSE2-NEXT: movdqa %xmm8, 256(%rdi) 1474; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1475; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1476; SSE2-NEXT: movdqa %xmm8, 240(%rdi) 1477; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1478; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1479; SSE2-NEXT: movdqa %xmm8, 224(%rdi) 1480; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1481; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1482; SSE2-NEXT: movdqa %xmm8, 208(%rdi) 1483; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1484; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1485; SSE2-NEXT: movdqa %xmm8, 192(%rdi) 1486; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1487; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1488; SSE2-NEXT: movdqa %xmm8, 176(%rdi) 1489; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1490; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1491; SSE2-NEXT: movdqa %xmm8, 160(%rdi) 1492; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1493; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1494; SSE2-NEXT: movdqa %xmm8, 144(%rdi) 1495; SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8 1496; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm8 1497; SSE2-NEXT: movdqa %xmm8, 128(%rdi) 1498; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm7 1499; SSE2-NEXT: movdqa %xmm7, 112(%rdi) 1500; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm6 1501; SSE2-NEXT: movdqa %xmm6, 96(%rdi) 1502; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm5 1503; SSE2-NEXT: movdqa %xmm5, 80(%rdi) 1504; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm4 1505; SSE2-NEXT: movdqa %xmm4, 64(%rdi) 1506; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm3 1507; SSE2-NEXT: movdqa %xmm3, 48(%rdi) 1508; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm2 1509; SSE2-NEXT: movdqa %xmm2, 32(%rdi) 1510; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm1 1511; SSE2-NEXT: movdqa %xmm1, 16(%rdi) 1512; SSE2-NEXT: pavgb {{[0-9]+}}(%rsp), %xmm0 1513; SSE2-NEXT: movdqa %xmm0, (%rdi) 1514; SSE2-NEXT: retq 1515; 1516; AVX1-LABEL: avg_v512i8_3: 1517; AVX1: # %bb.0: 1518; AVX1-NEXT: pushq %rbp 1519; AVX1-NEXT: movq %rsp, %rbp 1520; AVX1-NEXT: andq $-32, %rsp 1521; AVX1-NEXT: subq $32, %rsp 1522; AVX1-NEXT: movq %rdi, %rax 1523; AVX1-NEXT: vmovdqa 256(%rbp), %xmm8 1524; AVX1-NEXT: vpavgb 768(%rbp), %xmm8, %xmm8 1525; AVX1-NEXT: vmovdqa %xmm8, 496(%rdi) 1526; AVX1-NEXT: vmovdqa 240(%rbp), %xmm8 1527; AVX1-NEXT: vpavgb 752(%rbp), %xmm8, %xmm8 1528; AVX1-NEXT: vmovdqa %xmm8, 480(%rdi) 1529; AVX1-NEXT: vmovdqa 224(%rbp), %xmm8 1530; AVX1-NEXT: vpavgb 736(%rbp), %xmm8, %xmm8 1531; AVX1-NEXT: vmovdqa %xmm8, 464(%rdi) 1532; AVX1-NEXT: vmovdqa 208(%rbp), %xmm8 1533; AVX1-NEXT: vpavgb 720(%rbp), %xmm8, %xmm8 1534; AVX1-NEXT: vmovdqa %xmm8, 448(%rdi) 1535; AVX1-NEXT: vmovdqa 192(%rbp), %xmm8 1536; AVX1-NEXT: vpavgb 704(%rbp), %xmm8, %xmm8 1537; AVX1-NEXT: vmovdqa %xmm8, 432(%rdi) 1538; AVX1-NEXT: vmovdqa 176(%rbp), %xmm8 1539; AVX1-NEXT: vpavgb 688(%rbp), %xmm8, %xmm8 1540; AVX1-NEXT: vmovdqa %xmm8, 416(%rdi) 1541; AVX1-NEXT: vmovdqa 160(%rbp), %xmm8 1542; AVX1-NEXT: vpavgb 672(%rbp), %xmm8, %xmm8 1543; AVX1-NEXT: vmovdqa %xmm8, 400(%rdi) 1544; AVX1-NEXT: vmovdqa 144(%rbp), %xmm8 1545; AVX1-NEXT: vpavgb 656(%rbp), %xmm8, %xmm8 1546; AVX1-NEXT: vmovdqa %xmm8, 384(%rdi) 1547; AVX1-NEXT: vmovdqa 128(%rbp), %xmm8 1548; AVX1-NEXT: vpavgb 640(%rbp), %xmm8, %xmm8 1549; AVX1-NEXT: vmovdqa %xmm8, 368(%rdi) 1550; AVX1-NEXT: vmovdqa 112(%rbp), %xmm8 1551; AVX1-NEXT: vpavgb 624(%rbp), %xmm8, %xmm8 1552; AVX1-NEXT: vmovdqa %xmm8, 352(%rdi) 1553; AVX1-NEXT: vmovdqa 96(%rbp), %xmm8 1554; AVX1-NEXT: vpavgb 608(%rbp), %xmm8, %xmm8 1555; AVX1-NEXT: vmovdqa %xmm8, 336(%rdi) 1556; AVX1-NEXT: vmovdqa 80(%rbp), %xmm8 1557; AVX1-NEXT: vpavgb 592(%rbp), %xmm8, %xmm8 1558; AVX1-NEXT: vmovdqa %xmm8, 320(%rdi) 1559; AVX1-NEXT: vmovdqa 64(%rbp), %xmm8 1560; AVX1-NEXT: vpavgb 576(%rbp), %xmm8, %xmm8 1561; AVX1-NEXT: vmovdqa %xmm8, 304(%rdi) 1562; AVX1-NEXT: vmovdqa 48(%rbp), %xmm8 1563; AVX1-NEXT: vpavgb 560(%rbp), %xmm8, %xmm8 1564; AVX1-NEXT: vmovdqa %xmm8, 288(%rdi) 1565; AVX1-NEXT: vmovdqa 32(%rbp), %xmm8 1566; AVX1-NEXT: vpavgb 544(%rbp), %xmm8, %xmm8 1567; AVX1-NEXT: vmovdqa %xmm8, 272(%rdi) 1568; AVX1-NEXT: vmovdqa 16(%rbp), %xmm8 1569; AVX1-NEXT: vpavgb 528(%rbp), %xmm8, %xmm8 1570; AVX1-NEXT: vmovdqa %xmm8, 256(%rdi) 1571; AVX1-NEXT: vextractf128 $1, %ymm7, %xmm8 1572; AVX1-NEXT: vpavgb 512(%rbp), %xmm8, %xmm8 1573; AVX1-NEXT: vmovdqa %xmm8, 240(%rdi) 1574; AVX1-NEXT: vpavgb 496(%rbp), %xmm7, %xmm7 1575; AVX1-NEXT: vmovdqa %xmm7, 224(%rdi) 1576; AVX1-NEXT: vextractf128 $1, %ymm6, %xmm7 1577; AVX1-NEXT: vpavgb 480(%rbp), %xmm7, %xmm7 1578; AVX1-NEXT: vmovdqa %xmm7, 208(%rdi) 1579; AVX1-NEXT: vpavgb 464(%rbp), %xmm6, %xmm6 1580; AVX1-NEXT: vmovdqa %xmm6, 192(%rdi) 1581; AVX1-NEXT: vextractf128 $1, %ymm5, %xmm6 1582; AVX1-NEXT: vpavgb 448(%rbp), %xmm6, %xmm6 1583; AVX1-NEXT: vmovdqa %xmm6, 176(%rdi) 1584; AVX1-NEXT: vpavgb 432(%rbp), %xmm5, %xmm5 1585; AVX1-NEXT: vmovdqa %xmm5, 160(%rdi) 1586; AVX1-NEXT: vextractf128 $1, %ymm4, %xmm5 1587; AVX1-NEXT: vpavgb 416(%rbp), %xmm5, %xmm5 1588; AVX1-NEXT: vmovdqa %xmm5, 144(%rdi) 1589; AVX1-NEXT: vpavgb 400(%rbp), %xmm4, %xmm4 1590; AVX1-NEXT: vmovdqa %xmm4, 128(%rdi) 1591; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm4 1592; AVX1-NEXT: vpavgb 384(%rbp), %xmm4, %xmm4 1593; AVX1-NEXT: vmovdqa %xmm4, 112(%rdi) 1594; AVX1-NEXT: vpavgb 368(%rbp), %xmm3, %xmm3 1595; AVX1-NEXT: vmovdqa %xmm3, 96(%rdi) 1596; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm3 1597; AVX1-NEXT: vpavgb 352(%rbp), %xmm3, %xmm3 1598; AVX1-NEXT: vmovdqa %xmm3, 80(%rdi) 1599; AVX1-NEXT: vpavgb 336(%rbp), %xmm2, %xmm2 1600; AVX1-NEXT: vmovdqa %xmm2, 64(%rdi) 1601; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 1602; AVX1-NEXT: vpavgb 320(%rbp), %xmm2, %xmm2 1603; AVX1-NEXT: vmovdqa %xmm2, 48(%rdi) 1604; AVX1-NEXT: vpavgb 304(%rbp), %xmm1, %xmm1 1605; AVX1-NEXT: vmovdqa %xmm1, 32(%rdi) 1606; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 1607; AVX1-NEXT: vpavgb 288(%rbp), %xmm1, %xmm1 1608; AVX1-NEXT: vmovdqa %xmm1, 16(%rdi) 1609; AVX1-NEXT: vpavgb 272(%rbp), %xmm0, %xmm0 1610; AVX1-NEXT: vmovdqa %xmm0, (%rdi) 1611; AVX1-NEXT: movq %rbp, %rsp 1612; AVX1-NEXT: popq %rbp 1613; AVX1-NEXT: vzeroupper 1614; AVX1-NEXT: retq 1615; 1616; AVX2-LABEL: avg_v512i8_3: 1617; AVX2: # %bb.0: 1618; AVX2-NEXT: pushq %rbp 1619; AVX2-NEXT: movq %rsp, %rbp 1620; AVX2-NEXT: andq $-32, %rsp 1621; AVX2-NEXT: subq $32, %rsp 1622; AVX2-NEXT: movq %rdi, %rax 1623; AVX2-NEXT: vmovdqa 240(%rbp), %ymm8 1624; AVX2-NEXT: vmovdqa 208(%rbp), %ymm9 1625; AVX2-NEXT: vmovdqa 176(%rbp), %ymm10 1626; AVX2-NEXT: vmovdqa 144(%rbp), %ymm11 1627; AVX2-NEXT: vmovdqa 112(%rbp), %ymm12 1628; AVX2-NEXT: vmovdqa 80(%rbp), %ymm13 1629; AVX2-NEXT: vmovdqa 48(%rbp), %ymm14 1630; AVX2-NEXT: vmovdqa 16(%rbp), %ymm15 1631; AVX2-NEXT: vpavgb 272(%rbp), %ymm0, %ymm0 1632; AVX2-NEXT: vpavgb 304(%rbp), %ymm1, %ymm1 1633; AVX2-NEXT: vpavgb 336(%rbp), %ymm2, %ymm2 1634; AVX2-NEXT: vpavgb 368(%rbp), %ymm3, %ymm3 1635; AVX2-NEXT: vpavgb 400(%rbp), %ymm4, %ymm4 1636; AVX2-NEXT: vpavgb 432(%rbp), %ymm5, %ymm5 1637; AVX2-NEXT: vpavgb 464(%rbp), %ymm6, %ymm6 1638; AVX2-NEXT: vpavgb 496(%rbp), %ymm7, %ymm7 1639; AVX2-NEXT: vpavgb 528(%rbp), %ymm15, %ymm15 1640; AVX2-NEXT: vpavgb 560(%rbp), %ymm14, %ymm14 1641; AVX2-NEXT: vpavgb 592(%rbp), %ymm13, %ymm13 1642; AVX2-NEXT: vpavgb 624(%rbp), %ymm12, %ymm12 1643; AVX2-NEXT: vpavgb 656(%rbp), %ymm11, %ymm11 1644; AVX2-NEXT: vpavgb 688(%rbp), %ymm10, %ymm10 1645; AVX2-NEXT: vpavgb 720(%rbp), %ymm9, %ymm9 1646; AVX2-NEXT: vpavgb 752(%rbp), %ymm8, %ymm8 1647; AVX2-NEXT: vmovdqa %ymm8, 480(%rdi) 1648; AVX2-NEXT: vmovdqa %ymm9, 448(%rdi) 1649; AVX2-NEXT: vmovdqa %ymm10, 416(%rdi) 1650; AVX2-NEXT: vmovdqa %ymm11, 384(%rdi) 1651; AVX2-NEXT: vmovdqa %ymm12, 352(%rdi) 1652; AVX2-NEXT: vmovdqa %ymm13, 320(%rdi) 1653; AVX2-NEXT: vmovdqa %ymm14, 288(%rdi) 1654; AVX2-NEXT: vmovdqa %ymm15, 256(%rdi) 1655; AVX2-NEXT: vmovdqa %ymm7, 224(%rdi) 1656; AVX2-NEXT: vmovdqa %ymm6, 192(%rdi) 1657; AVX2-NEXT: vmovdqa %ymm5, 160(%rdi) 1658; AVX2-NEXT: vmovdqa %ymm4, 128(%rdi) 1659; AVX2-NEXT: vmovdqa %ymm3, 96(%rdi) 1660; AVX2-NEXT: vmovdqa %ymm2, 64(%rdi) 1661; AVX2-NEXT: vmovdqa %ymm1, 32(%rdi) 1662; AVX2-NEXT: vmovdqa %ymm0, (%rdi) 1663; AVX2-NEXT: movq %rbp, %rsp 1664; AVX2-NEXT: popq %rbp 1665; AVX2-NEXT: vzeroupper 1666; AVX2-NEXT: retq 1667; 1668; AVX512F-LABEL: avg_v512i8_3: 1669; AVX512F: # %bb.0: 1670; AVX512F-NEXT: pushq %rbp 1671; AVX512F-NEXT: movq %rsp, %rbp 1672; AVX512F-NEXT: andq $-64, %rsp 1673; AVX512F-NEXT: subq $64, %rsp 1674; AVX512F-NEXT: movq %rdi, %rax 1675; AVX512F-NEXT: vpavgb 16(%rbp), %ymm0, %ymm8 1676; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm0 1677; AVX512F-NEXT: vpavgb 48(%rbp), %ymm0, %ymm0 1678; AVX512F-NEXT: vpavgb 80(%rbp), %ymm1, %ymm9 1679; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm1 1680; AVX512F-NEXT: vpavgb 112(%rbp), %ymm1, %ymm1 1681; AVX512F-NEXT: vpavgb 144(%rbp), %ymm2, %ymm10 1682; AVX512F-NEXT: vextracti64x4 $1, %zmm2, %ymm2 1683; AVX512F-NEXT: vpavgb 176(%rbp), %ymm2, %ymm2 1684; AVX512F-NEXT: vpavgb 208(%rbp), %ymm3, %ymm11 1685; AVX512F-NEXT: vextracti64x4 $1, %zmm3, %ymm3 1686; AVX512F-NEXT: vpavgb 240(%rbp), %ymm3, %ymm3 1687; AVX512F-NEXT: vpavgb 272(%rbp), %ymm4, %ymm12 1688; AVX512F-NEXT: vextracti64x4 $1, %zmm4, %ymm4 1689; AVX512F-NEXT: vpavgb 304(%rbp), %ymm4, %ymm4 1690; AVX512F-NEXT: vpavgb 336(%rbp), %ymm5, %ymm13 1691; AVX512F-NEXT: vextracti64x4 $1, %zmm5, %ymm5 1692; AVX512F-NEXT: vpavgb 368(%rbp), %ymm5, %ymm5 1693; AVX512F-NEXT: vpavgb 400(%rbp), %ymm6, %ymm14 1694; AVX512F-NEXT: vextracti64x4 $1, %zmm6, %ymm6 1695; AVX512F-NEXT: vpavgb 432(%rbp), %ymm6, %ymm6 1696; AVX512F-NEXT: vpavgb 464(%rbp), %ymm7, %ymm15 1697; AVX512F-NEXT: vextracti64x4 $1, %zmm7, %ymm7 1698; AVX512F-NEXT: vpavgb 496(%rbp), %ymm7, %ymm7 1699; AVX512F-NEXT: vmovdqa %ymm7, 480(%rdi) 1700; AVX512F-NEXT: vmovdqa %ymm15, 448(%rdi) 1701; AVX512F-NEXT: vmovdqa %ymm6, 416(%rdi) 1702; AVX512F-NEXT: vmovdqa %ymm14, 384(%rdi) 1703; AVX512F-NEXT: vmovdqa %ymm5, 352(%rdi) 1704; AVX512F-NEXT: vmovdqa %ymm13, 320(%rdi) 1705; AVX512F-NEXT: vmovdqa %ymm4, 288(%rdi) 1706; AVX512F-NEXT: vmovdqa %ymm12, 256(%rdi) 1707; AVX512F-NEXT: vmovdqa %ymm3, 224(%rdi) 1708; AVX512F-NEXT: vmovdqa %ymm11, 192(%rdi) 1709; AVX512F-NEXT: vmovdqa %ymm2, 160(%rdi) 1710; AVX512F-NEXT: vmovdqa %ymm10, 128(%rdi) 1711; AVX512F-NEXT: vmovdqa %ymm1, 96(%rdi) 1712; AVX512F-NEXT: vmovdqa %ymm9, 64(%rdi) 1713; AVX512F-NEXT: vmovdqa %ymm0, 32(%rdi) 1714; AVX512F-NEXT: vmovdqa %ymm8, (%rdi) 1715; AVX512F-NEXT: movq %rbp, %rsp 1716; AVX512F-NEXT: popq %rbp 1717; AVX512F-NEXT: vzeroupper 1718; AVX512F-NEXT: retq 1719; 1720; AVX512BW-LABEL: avg_v512i8_3: 1721; AVX512BW: # %bb.0: 1722; AVX512BW-NEXT: pushq %rbp 1723; AVX512BW-NEXT: movq %rsp, %rbp 1724; AVX512BW-NEXT: andq $-64, %rsp 1725; AVX512BW-NEXT: subq $64, %rsp 1726; AVX512BW-NEXT: movq %rdi, %rax 1727; AVX512BW-NEXT: vpavgb 16(%rbp), %zmm0, %zmm0 1728; AVX512BW-NEXT: vpavgb 80(%rbp), %zmm1, %zmm1 1729; AVX512BW-NEXT: vpavgb 144(%rbp), %zmm2, %zmm2 1730; AVX512BW-NEXT: vpavgb 208(%rbp), %zmm3, %zmm3 1731; AVX512BW-NEXT: vpavgb 272(%rbp), %zmm4, %zmm4 1732; AVX512BW-NEXT: vpavgb 336(%rbp), %zmm5, %zmm5 1733; AVX512BW-NEXT: vpavgb 400(%rbp), %zmm6, %zmm6 1734; AVX512BW-NEXT: vpavgb 464(%rbp), %zmm7, %zmm7 1735; AVX512BW-NEXT: vmovdqa64 %zmm7, 448(%rdi) 1736; AVX512BW-NEXT: vmovdqa64 %zmm6, 384(%rdi) 1737; AVX512BW-NEXT: vmovdqa64 %zmm5, 320(%rdi) 1738; AVX512BW-NEXT: vmovdqa64 %zmm4, 256(%rdi) 1739; AVX512BW-NEXT: vmovdqa64 %zmm3, 192(%rdi) 1740; AVX512BW-NEXT: vmovdqa64 %zmm2, 128(%rdi) 1741; AVX512BW-NEXT: vmovdqa64 %zmm1, 64(%rdi) 1742; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdi) 1743; AVX512BW-NEXT: movq %rbp, %rsp 1744; AVX512BW-NEXT: popq %rbp 1745; AVX512BW-NEXT: vzeroupper 1746; AVX512BW-NEXT: retq 1747 %za = zext <512 x i8> %a to <512 x i16> 1748 %zb = zext <512 x i8> %b to <512 x i16> 1749 %add = add nuw nsw <512 x i16> %za, %zb 1750 %add1 = add nuw nsw <512 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> 1751 %lshr = lshr <512 x i16> %add1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> 1752 %res = trunc <512 x i16> %lshr to <512 x i8> 1753 ret <512 x i8> %res 1754} 1755 1756; This is not an avg, but its structurally similar and previously caused a crash 1757; because the constants can't be read with APInt::getZExtValue. 1758define void @not_avg_v16i8_wide_constants(ptr %a, ptr %b) nounwind { 1759; SSE2-LABEL: not_avg_v16i8_wide_constants: 1760; SSE2: # %bb.0: 1761; SSE2-NEXT: pushq %rbp 1762; SSE2-NEXT: pushq %r15 1763; SSE2-NEXT: pushq %r14 1764; SSE2-NEXT: pushq %r13 1765; SSE2-NEXT: pushq %r12 1766; SSE2-NEXT: pushq %rbx 1767; SSE2-NEXT: movaps (%rdi), %xmm1 1768; SSE2-NEXT: movaps (%rsi), %xmm0 1769; SSE2-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp) 1770; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax 1771; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 1772; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax 1773; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 1774; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax 1775; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 1776; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax 1777; SSE2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 1778; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %r15d 1779; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %r14d 1780; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %r12d 1781; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %r13d 1782; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %r10d 1783; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %ebx 1784; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %r9d 1785; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %r8d 1786; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %edi 1787; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx 1788; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi 1789; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %ebp 1790; SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp) 1791; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %ecx 1792; SSE2-NEXT: addq %rsi, %rcx 1793; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %eax 1794; SSE2-NEXT: addq %rbp, %rax 1795; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %esi 1796; SSE2-NEXT: leaq -1(%rdx,%rsi), %r11 1797; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx 1798; SSE2-NEXT: leaq -1(%rdi,%rdx), %rsi 1799; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx 1800; SSE2-NEXT: leaq -1(%r8,%rdx), %rdi 1801; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx 1802; SSE2-NEXT: leaq -1(%r9,%rdx), %r8 1803; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx 1804; SSE2-NEXT: leaq -1(%rbx,%rdx), %rbx 1805; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx 1806; SSE2-NEXT: leaq -1(%r10,%rdx), %r9 1807; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx 1808; SSE2-NEXT: leaq -1(%r13,%rdx), %r13 1809; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx 1810; SSE2-NEXT: leaq -1(%r12,%rdx), %r12 1811; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx 1812; SSE2-NEXT: leaq -1(%r14,%rdx), %r14 1813; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx 1814; SSE2-NEXT: leaq -1(%r15,%rdx), %r15 1815; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx 1816; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload 1817; SSE2-NEXT: leaq -1(%rbp,%rdx), %rdx 1818; SSE2-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 1819; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx 1820; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload 1821; SSE2-NEXT: leaq -1(%rbp,%rdx), %r10 1822; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx 1823; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload 1824; SSE2-NEXT: leaq -1(%rbp,%rdx), %rdx 1825; SSE2-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 1826; SSE2-NEXT: movzbl -{{[0-9]+}}(%rsp), %edx 1827; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload 1828; SSE2-NEXT: leaq -1(%rbp,%rdx), %rdx 1829; SSE2-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 1830; SSE2-NEXT: xorl %ebp, %ebp 1831; SSE2-NEXT: addq $-1, %rcx 1832; SSE2-NEXT: movl $0, %edx 1833; SSE2-NEXT: adcq $-1, %rdx 1834; SSE2-NEXT: addq $-1, %rax 1835; SSE2-NEXT: adcq $-1, %rbp 1836; SSE2-NEXT: shldq $63, %rax, %rbp 1837; SSE2-NEXT: shldq $63, %rcx, %rdx 1838; SSE2-NEXT: movq %rdx, %xmm8 1839; SSE2-NEXT: movq %rbp, %xmm0 1840; SSE2-NEXT: shrq %r11 1841; SSE2-NEXT: movq %r11, %xmm9 1842; SSE2-NEXT: shrq %rsi 1843; SSE2-NEXT: movq %rsi, %xmm2 1844; SSE2-NEXT: shrq %rdi 1845; SSE2-NEXT: movq %rdi, %xmm10 1846; SSE2-NEXT: shrq %r8 1847; SSE2-NEXT: movq %r8, %xmm4 1848; SSE2-NEXT: shrq %rbx 1849; SSE2-NEXT: movq %rbx, %xmm11 1850; SSE2-NEXT: shrq %r9 1851; SSE2-NEXT: movq %r9, %xmm7 1852; SSE2-NEXT: shrq %r13 1853; SSE2-NEXT: movq %r13, %xmm12 1854; SSE2-NEXT: shrq %r12 1855; SSE2-NEXT: movq %r12, %xmm1 1856; SSE2-NEXT: shrq %r14 1857; SSE2-NEXT: movq %r14, %xmm13 1858; SSE2-NEXT: shrq %r15 1859; SSE2-NEXT: movq %r15, %xmm6 1860; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload 1861; SSE2-NEXT: shrq %rax 1862; SSE2-NEXT: movq %rax, %xmm14 1863; SSE2-NEXT: shrq %r10 1864; SSE2-NEXT: movq %r10, %xmm5 1865; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload 1866; SSE2-NEXT: shrq %rax 1867; SSE2-NEXT: movq %rax, %xmm15 1868; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload 1869; SSE2-NEXT: shrq %rax 1870; SSE2-NEXT: movq %rax, %xmm3 1871; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm8[0],xmm0[1],xmm8[1],xmm0[2],xmm8[2],xmm0[3],xmm8[3],xmm0[4],xmm8[4],xmm0[5],xmm8[5],xmm0[6],xmm8[6],xmm0[7],xmm8[7] 1872; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3],xmm2[4],xmm9[4],xmm2[5],xmm9[5],xmm2[6],xmm9[6],xmm2[7],xmm9[7] 1873; SSE2-NEXT: movdqa {{.*#+}} xmm8 = [65535,0,65535,65535,65535,65535,65535,65535] 1874; SSE2-NEXT: pand %xmm8, %xmm0 1875; SSE2-NEXT: pslld $16, %xmm2 1876; SSE2-NEXT: pandn %xmm2, %xmm8 1877; SSE2-NEXT: por %xmm0, %xmm8 1878; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm10[0],xmm4[1],xmm10[1],xmm4[2],xmm10[2],xmm4[3],xmm10[3],xmm4[4],xmm10[4],xmm4[5],xmm10[5],xmm4[6],xmm10[6],xmm4[7],xmm10[7] 1879; SSE2-NEXT: psllq $48, %xmm4 1880; SSE2-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm11[0],xmm7[1],xmm11[1],xmm7[2],xmm11[2],xmm7[3],xmm11[3],xmm7[4],xmm11[4],xmm7[5],xmm11[5],xmm7[6],xmm11[6],xmm7[7],xmm11[7] 1881; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [65535,65535,65535,0,65535,65535,65535,65535] 1882; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm7[0,0,1,1] 1883; SSE2-NEXT: pand %xmm0, %xmm2 1884; SSE2-NEXT: pandn %xmm4, %xmm0 1885; SSE2-NEXT: por %xmm2, %xmm0 1886; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1] 1887; SSE2-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1] 1888; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm12[0],xmm1[1],xmm12[1],xmm1[2],xmm12[2],xmm1[3],xmm12[3],xmm1[4],xmm12[4],xmm1[5],xmm12[5],xmm1[6],xmm12[6],xmm1[7],xmm12[7] 1889; SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm13[0],xmm6[1],xmm13[1],xmm6[2],xmm13[2],xmm6[3],xmm13[3],xmm6[4],xmm13[4],xmm6[5],xmm13[5],xmm6[6],xmm13[6],xmm6[7],xmm13[7] 1890; SSE2-NEXT: pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1] 1891; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,0,0] 1892; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 1893; SSE2-NEXT: por %xmm1, %xmm0 1894; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm14[0],xmm5[1],xmm14[1],xmm5[2],xmm14[2],xmm5[3],xmm14[3],xmm5[4],xmm14[4],xmm5[5],xmm14[5],xmm5[6],xmm14[6],xmm5[7],xmm14[7] 1895; SSE2-NEXT: pslldq {{.*#+}} xmm5 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm5[0,1,2,3,4,5] 1896; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [65535,65535,65535,65535,65535,0,65535,65535] 1897; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm15[0],xmm3[1],xmm15[1],xmm3[2],xmm15[2],xmm3[3],xmm15[3],xmm3[4],xmm15[4],xmm3[5],xmm15[5],xmm3[6],xmm15[6],xmm3[7],xmm15[7] 1898; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[0,1,0,1] 1899; SSE2-NEXT: pand %xmm1, %xmm2 1900; SSE2-NEXT: pandn %xmm5, %xmm1 1901; SSE2-NEXT: por %xmm2, %xmm1 1902; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,2,2,2] 1903; SSE2-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm0[2],xmm1[3],xmm0[3] 1904; SSE2-NEXT: movsd {{.*#+}} xmm1 = xmm8[0],xmm1[1] 1905; SSE2-NEXT: movupd %xmm1, (%rax) 1906; SSE2-NEXT: popq %rbx 1907; SSE2-NEXT: popq %r12 1908; SSE2-NEXT: popq %r13 1909; SSE2-NEXT: popq %r14 1910; SSE2-NEXT: popq %r15 1911; SSE2-NEXT: popq %rbp 1912; SSE2-NEXT: retq 1913; 1914; AVX1-LABEL: not_avg_v16i8_wide_constants: 1915; AVX1: # %bb.0: 1916; AVX1-NEXT: pushq %rbp 1917; AVX1-NEXT: pushq %r15 1918; AVX1-NEXT: pushq %r14 1919; AVX1-NEXT: pushq %r13 1920; AVX1-NEXT: pushq %r12 1921; AVX1-NEXT: pushq %rbx 1922; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 1923; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 1924; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 1925; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero 1926; AVX1-NEXT: vpextrw $4, %xmm0, %eax 1927; AVX1-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 1928; AVX1-NEXT: vpextrw $5, %xmm0, %eax 1929; AVX1-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 1930; AVX1-NEXT: vpextrw $6, %xmm0, %eax 1931; AVX1-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 1932; AVX1-NEXT: vpextrw $7, %xmm0, %r11d 1933; AVX1-NEXT: vpextrw $0, %xmm3, %r14d 1934; AVX1-NEXT: vpextrw $1, %xmm3, %r15d 1935; AVX1-NEXT: vpextrw $2, %xmm3, %r10d 1936; AVX1-NEXT: vpextrw $3, %xmm3, %r9d 1937; AVX1-NEXT: vpextrw $4, %xmm3, %r8d 1938; AVX1-NEXT: vpextrw $5, %xmm3, %ebx 1939; AVX1-NEXT: vpextrw $6, %xmm3, %ebp 1940; AVX1-NEXT: vpextrw $7, %xmm3, %edi 1941; AVX1-NEXT: vpextrw $1, %xmm0, %edx 1942; AVX1-NEXT: vpextrw $0, %xmm0, %esi 1943; AVX1-NEXT: vpextrw $1, %xmm1, %ecx 1944; AVX1-NEXT: addq %rdx, %rcx 1945; AVX1-NEXT: vpextrw $0, %xmm1, %eax 1946; AVX1-NEXT: addq %rsi, %rax 1947; AVX1-NEXT: vpextrw $7, %xmm2, %edx 1948; AVX1-NEXT: leaq -1(%rdi,%rdx), %rdi 1949; AVX1-NEXT: vpextrw $6, %xmm2, %edx 1950; AVX1-NEXT: leaq -1(%rbp,%rdx), %rbp 1951; AVX1-NEXT: vpextrw $5, %xmm2, %edx 1952; AVX1-NEXT: leaq -1(%rbx,%rdx), %rbx 1953; AVX1-NEXT: vpextrw $4, %xmm2, %edx 1954; AVX1-NEXT: leaq -1(%r8,%rdx), %r8 1955; AVX1-NEXT: vpextrw $3, %xmm2, %edx 1956; AVX1-NEXT: leaq -1(%r9,%rdx), %r9 1957; AVX1-NEXT: vpextrw $2, %xmm2, %edx 1958; AVX1-NEXT: leaq -1(%r10,%rdx), %r10 1959; AVX1-NEXT: vpextrw $1, %xmm2, %edx 1960; AVX1-NEXT: leaq -1(%r15,%rdx), %r13 1961; AVX1-NEXT: vpextrw $0, %xmm2, %edx 1962; AVX1-NEXT: leaq -1(%r14,%rdx), %r12 1963; AVX1-NEXT: vpextrw $7, %xmm1, %edx 1964; AVX1-NEXT: leaq -1(%r11,%rdx), %r15 1965; AVX1-NEXT: vpextrw $6, %xmm1, %edx 1966; AVX1-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload 1967; AVX1-NEXT: leaq -1(%rsi,%rdx), %r14 1968; AVX1-NEXT: vpextrw $5, %xmm1, %edx 1969; AVX1-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload 1970; AVX1-NEXT: leaq -1(%rsi,%rdx), %r11 1971; AVX1-NEXT: vpextrw $4, %xmm1, %edx 1972; AVX1-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload 1973; AVX1-NEXT: leaq -1(%rsi,%rdx), %rdx 1974; AVX1-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 1975; AVX1-NEXT: vpextrw $3, %xmm0, %edx 1976; AVX1-NEXT: vpextrw $3, %xmm1, %esi 1977; AVX1-NEXT: leaq -1(%rdx,%rsi), %rdx 1978; AVX1-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 1979; AVX1-NEXT: vpextrw $2, %xmm0, %edx 1980; AVX1-NEXT: vpextrw $2, %xmm1, %esi 1981; AVX1-NEXT: leaq -1(%rdx,%rsi), %rdx 1982; AVX1-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 1983; AVX1-NEXT: xorl %edx, %edx 1984; AVX1-NEXT: addq $-1, %rcx 1985; AVX1-NEXT: movl $0, %esi 1986; AVX1-NEXT: adcq $-1, %rsi 1987; AVX1-NEXT: addq $-1, %rax 1988; AVX1-NEXT: adcq $-1, %rdx 1989; AVX1-NEXT: shldq $63, %rax, %rdx 1990; AVX1-NEXT: shldq $63, %rcx, %rsi 1991; AVX1-NEXT: shrq %rdi 1992; AVX1-NEXT: vmovq %rdi, %xmm8 1993; AVX1-NEXT: shrq %rbp 1994; AVX1-NEXT: vmovq %rbp, %xmm9 1995; AVX1-NEXT: shrq %rbx 1996; AVX1-NEXT: vmovq %rbx, %xmm0 1997; AVX1-NEXT: shrq %r8 1998; AVX1-NEXT: vmovq %r8, %xmm1 1999; AVX1-NEXT: shrq %r9 2000; AVX1-NEXT: vmovq %r9, %xmm12 2001; AVX1-NEXT: shrq %r10 2002; AVX1-NEXT: vmovq %r10, %xmm13 2003; AVX1-NEXT: shrq %r13 2004; AVX1-NEXT: vmovq %r13, %xmm14 2005; AVX1-NEXT: shrq %r12 2006; AVX1-NEXT: vmovq %r12, %xmm15 2007; AVX1-NEXT: shrq %r15 2008; AVX1-NEXT: vmovq %r15, %xmm10 2009; AVX1-NEXT: shrq %r14 2010; AVX1-NEXT: vmovq %r14, %xmm11 2011; AVX1-NEXT: shrq %r11 2012; AVX1-NEXT: vmovq %r11, %xmm2 2013; AVX1-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload 2014; AVX1-NEXT: shrq %rax 2015; AVX1-NEXT: vmovq %rax, %xmm3 2016; AVX1-NEXT: vmovq %rsi, %xmm4 2017; AVX1-NEXT: vmovq %rdx, %xmm5 2018; AVX1-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload 2019; AVX1-NEXT: shrq %rax 2020; AVX1-NEXT: vmovq %rax, %xmm6 2021; AVX1-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload 2022; AVX1-NEXT: shrq %rax 2023; AVX1-NEXT: vmovq %rax, %xmm7 2024; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3],xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7] 2025; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm9 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] 2026; AVX1-NEXT: vpslldq {{.*#+}} xmm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm8[0,1] 2027; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm9[0,0,0,0] 2028; AVX1-NEXT: vpblendw {{.*#+}} xmm8 = xmm0[0,1,2,3,4,5,6],xmm8[7] 2029; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3],xmm13[4],xmm12[4],xmm13[5],xmm12[5],xmm13[6],xmm12[6],xmm13[7],xmm12[7] 2030; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm9 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3],xmm15[4],xmm14[4],xmm15[5],xmm14[5],xmm15[6],xmm14[6],xmm15[7],xmm14[7] 2031; AVX1-NEXT: vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5] 2032; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm9[0,1,0,1] 2033; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3,4],xmm0[5],xmm1[6,7] 2034; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm8[6,7] 2035; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3],xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7] 2036; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] 2037; AVX1-NEXT: vpsllq $48, %xmm1, %xmm1 2038; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,0,1,1] 2039; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3],xmm2[4,5,6,7] 2040; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7] 2041; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm3 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3],xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7] 2042; AVX1-NEXT: vpslld $16, %xmm3, %xmm3 2043; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2,3,4,5,6,7] 2044; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5,6,7] 2045; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7] 2046; AVX1-NEXT: vmovdqu %xmm0, (%rax) 2047; AVX1-NEXT: popq %rbx 2048; AVX1-NEXT: popq %r12 2049; AVX1-NEXT: popq %r13 2050; AVX1-NEXT: popq %r14 2051; AVX1-NEXT: popq %r15 2052; AVX1-NEXT: popq %rbp 2053; AVX1-NEXT: retq 2054; 2055; AVX2-LABEL: not_avg_v16i8_wide_constants: 2056; AVX2: # %bb.0: 2057; AVX2-NEXT: pushq %rbp 2058; AVX2-NEXT: pushq %r15 2059; AVX2-NEXT: pushq %r14 2060; AVX2-NEXT: pushq %r13 2061; AVX2-NEXT: pushq %r12 2062; AVX2-NEXT: pushq %rbx 2063; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero 2064; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero 2065; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero 2066; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero 2067; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm9 2068; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm1 2069; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero 2070; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm5 2071; AVX2-NEXT: vmovq %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill 2072; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm2 2073; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero 2074; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm6 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero 2075; AVX2-NEXT: vmovq %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill 2076; AVX2-NEXT: vextracti128 $1, %ymm6, %xmm7 2077; AVX2-NEXT: vmovq %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill 2078; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm2 2079; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero 2080; AVX2-NEXT: vmovq %xmm2, %r11 2081; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm0 2082; AVX2-NEXT: vmovq %xmm0, %r14 2083; AVX2-NEXT: vpextrq $1, %xmm0, %rbx 2084; AVX2-NEXT: vpextrq $1, %xmm2, %rsi 2085; AVX2-NEXT: vpextrq $1, %xmm7, %r12 2086; AVX2-NEXT: vpextrq $1, %xmm6, %r15 2087; AVX2-NEXT: vpextrq $1, %xmm5, %rdx 2088; AVX2-NEXT: vpextrq $1, %xmm1, %rcx 2089; AVX2-NEXT: vpextrq $1, %xmm3, %rax 2090; AVX2-NEXT: vmovq %xmm3, %rbp 2091; AVX2-NEXT: vpextrq $1, %xmm9, %r9 2092; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero 2093; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm6 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero 2094; AVX2-NEXT: vextracti128 $1, %ymm6, %xmm2 2095; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0 2096; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero 2097; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm5 2098; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm0 2099; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero 2100; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero 2101; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm7 2102; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0 2103; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm8 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero 2104; AVX2-NEXT: vextracti128 $1, %ymm8, %xmm0 2105; AVX2-NEXT: vpextrq $1, %xmm0, %rdi 2106; AVX2-NEXT: addq %rbx, %rdi 2107; AVX2-NEXT: movq %rdi, %rbx 2108; AVX2-NEXT: vpextrq $1, %xmm8, %r10 2109; AVX2-NEXT: addq %rsi, %r10 2110; AVX2-NEXT: vpextrq $1, %xmm7, %rsi 2111; AVX2-NEXT: addq %r12, %rsi 2112; AVX2-NEXT: movq %rsi, %r12 2113; AVX2-NEXT: vpextrq $1, %xmm4, %r13 2114; AVX2-NEXT: addq %r15, %r13 2115; AVX2-NEXT: vpextrq $1, %xmm5, %r15 2116; AVX2-NEXT: addq %rdx, %r15 2117; AVX2-NEXT: vpextrq $1, %xmm3, %r8 2118; AVX2-NEXT: addq %rcx, %r8 2119; AVX2-NEXT: vpextrq $1, %xmm6, %rsi 2120; AVX2-NEXT: addq %rax, %rsi 2121; AVX2-NEXT: vmovq %xmm6, %rdx 2122; AVX2-NEXT: addq %rbp, %rdx 2123; AVX2-NEXT: vpextrq $1, %xmm2, %rcx 2124; AVX2-NEXT: addq %r9, %rcx 2125; AVX2-NEXT: vmovq %xmm0, %rdi 2126; AVX2-NEXT: leaq -1(%r14,%rdi), %rax 2127; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2128; AVX2-NEXT: vmovq %xmm8, %rdi 2129; AVX2-NEXT: leaq -1(%r11,%rdi), %rax 2130; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2131; AVX2-NEXT: vmovq %xmm7, %rdi 2132; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload 2133; AVX2-NEXT: leaq -1(%rax,%rdi), %rax 2134; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2135; AVX2-NEXT: vmovq %xmm4, %rdi 2136; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload 2137; AVX2-NEXT: leaq -1(%rax,%rdi), %rax 2138; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2139; AVX2-NEXT: vmovq %xmm5, %rdi 2140; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload 2141; AVX2-NEXT: leaq -1(%rax,%rdi), %rax 2142; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2143; AVX2-NEXT: vmovq %xmm1, %rdi 2144; AVX2-NEXT: vmovq %xmm3, %rbp 2145; AVX2-NEXT: leaq -1(%rdi,%rbp), %rax 2146; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2147; AVX2-NEXT: vmovq %xmm9, %rdi 2148; AVX2-NEXT: vmovq %xmm2, %rbp 2149; AVX2-NEXT: leaq -1(%rdi,%rbp), %rdi 2150; AVX2-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2151; AVX2-NEXT: addq $-1, %rbx 2152; AVX2-NEXT: movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2153; AVX2-NEXT: movl $0, %r9d 2154; AVX2-NEXT: adcq $-1, %r9 2155; AVX2-NEXT: addq $-1, %r10 2156; AVX2-NEXT: movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2157; AVX2-NEXT: movl $0, %edi 2158; AVX2-NEXT: adcq $-1, %rdi 2159; AVX2-NEXT: addq $-1, %r12 2160; AVX2-NEXT: movq %r12, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2161; AVX2-NEXT: movl $0, %r11d 2162; AVX2-NEXT: adcq $-1, %r11 2163; AVX2-NEXT: addq $-1, %r13 2164; AVX2-NEXT: movl $0, %r10d 2165; AVX2-NEXT: adcq $-1, %r10 2166; AVX2-NEXT: addq $-1, %r15 2167; AVX2-NEXT: movl $0, %r14d 2168; AVX2-NEXT: adcq $-1, %r14 2169; AVX2-NEXT: addq $-1, %r8 2170; AVX2-NEXT: movl $0, %ebp 2171; AVX2-NEXT: adcq $-1, %rbp 2172; AVX2-NEXT: addq $-1, %rsi 2173; AVX2-NEXT: movl $0, %r12d 2174; AVX2-NEXT: adcq $-1, %r12 2175; AVX2-NEXT: addq $-1, %rdx 2176; AVX2-NEXT: movl $0, %ebx 2177; AVX2-NEXT: adcq $-1, %rbx 2178; AVX2-NEXT: addq $-1, %rcx 2179; AVX2-NEXT: movl $0, %eax 2180; AVX2-NEXT: adcq $-1, %rax 2181; AVX2-NEXT: shldq $63, %rcx, %rax 2182; AVX2-NEXT: shldq $63, %rdx, %rbx 2183; AVX2-NEXT: shldq $63, %rsi, %r12 2184; AVX2-NEXT: shldq $63, %r8, %rbp 2185; AVX2-NEXT: shldq $63, %r15, %r14 2186; AVX2-NEXT: shldq $63, %r13, %r10 2187; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload 2188; AVX2-NEXT: shldq $63, %rcx, %r11 2189; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload 2190; AVX2-NEXT: shldq $63, %rcx, %rdi 2191; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload 2192; AVX2-NEXT: shldq $63, %rcx, %r9 2193; AVX2-NEXT: vmovq %r9, %xmm8 2194; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload 2195; AVX2-NEXT: shrq %rcx 2196; AVX2-NEXT: vmovq %rcx, %xmm9 2197; AVX2-NEXT: vmovq %rdi, %xmm0 2198; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload 2199; AVX2-NEXT: shrq %rcx 2200; AVX2-NEXT: vmovq %rcx, %xmm1 2201; AVX2-NEXT: vmovq %r11, %xmm12 2202; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload 2203; AVX2-NEXT: shrq %rcx 2204; AVX2-NEXT: vmovq %rcx, %xmm13 2205; AVX2-NEXT: vmovq %r10, %xmm14 2206; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload 2207; AVX2-NEXT: shrq %rcx 2208; AVX2-NEXT: vmovq %rcx, %xmm15 2209; AVX2-NEXT: vmovq %r14, %xmm10 2210; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload 2211; AVX2-NEXT: shrq %rcx 2212; AVX2-NEXT: vmovq %rcx, %xmm11 2213; AVX2-NEXT: vmovq %rbp, %xmm2 2214; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload 2215; AVX2-NEXT: shrq %rcx 2216; AVX2-NEXT: vmovq %rcx, %xmm3 2217; AVX2-NEXT: vmovq %r12, %xmm4 2218; AVX2-NEXT: vmovq %rbx, %xmm5 2219; AVX2-NEXT: vmovq %rax, %xmm6 2220; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload 2221; AVX2-NEXT: shrq %rax 2222; AVX2-NEXT: vmovq %rax, %xmm7 2223; AVX2-NEXT: vpunpcklbw {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3],xmm9[4],xmm8[4],xmm9[5],xmm8[5],xmm9[6],xmm8[6],xmm9[7],xmm8[7] 2224; AVX2-NEXT: vpunpcklbw {{.*#+}} xmm9 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] 2225; AVX2-NEXT: vpbroadcastw %xmm8, %xmm8 2226; AVX2-NEXT: vpbroadcastw %xmm9, %xmm0 2227; AVX2-NEXT: vpunpcklwd {{.*#+}} xmm8 = xmm0[0],xmm8[0],xmm0[1],xmm8[1],xmm0[2],xmm8[2],xmm0[3],xmm8[3] 2228; AVX2-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3],xmm13[4],xmm12[4],xmm13[5],xmm12[5],xmm13[6],xmm12[6],xmm13[7],xmm12[7] 2229; AVX2-NEXT: vpunpcklbw {{.*#+}} xmm9 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3],xmm15[4],xmm14[4],xmm15[5],xmm14[5],xmm15[6],xmm14[6],xmm15[7],xmm14[7] 2230; AVX2-NEXT: vpbroadcastw %xmm0, %xmm0 2231; AVX2-NEXT: vpbroadcastw %xmm9, %xmm1 2232; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3,4],xmm0[5],xmm1[6,7] 2233; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm8[3] 2234; AVX2-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3],xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7] 2235; AVX2-NEXT: vpsllq $48, %xmm1, %xmm1 2236; AVX2-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] 2237; AVX2-NEXT: vpbroadcastw %xmm2, %xmm2 2238; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3],xmm2[4,5,6,7] 2239; AVX2-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7] 2240; AVX2-NEXT: vpunpcklbw {{.*#+}} xmm3 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3],xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7] 2241; AVX2-NEXT: vpslld $16, %xmm3, %xmm3 2242; AVX2-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2,3,4,5,6,7] 2243; AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2,3] 2244; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3] 2245; AVX2-NEXT: vmovdqu %xmm0, (%rax) 2246; AVX2-NEXT: popq %rbx 2247; AVX2-NEXT: popq %r12 2248; AVX2-NEXT: popq %r13 2249; AVX2-NEXT: popq %r14 2250; AVX2-NEXT: popq %r15 2251; AVX2-NEXT: popq %rbp 2252; AVX2-NEXT: vzeroupper 2253; AVX2-NEXT: retq 2254; 2255; AVX512-LABEL: not_avg_v16i8_wide_constants: 2256; AVX512: # %bb.0: 2257; AVX512-NEXT: pushq %rbp 2258; AVX512-NEXT: pushq %r15 2259; AVX512-NEXT: pushq %r14 2260; AVX512-NEXT: pushq %r13 2261; AVX512-NEXT: pushq %r12 2262; AVX512-NEXT: pushq %rbx 2263; AVX512-NEXT: subq $24, %rsp 2264; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero 2265; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero 2266; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero 2267; AVX512-NEXT: vpmovzxdq {{.*#+}} ymm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero 2268; AVX512-NEXT: vextracti128 $1, %ymm3, %xmm4 2269; AVX512-NEXT: vmovq %xmm4, %rbx 2270; AVX512-NEXT: vpextrq $1, %xmm4, %rbp 2271; AVX512-NEXT: vmovq %xmm3, %rdi 2272; AVX512-NEXT: vpextrq $1, %xmm3, %rsi 2273; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm2 2274; AVX512-NEXT: vpmovzxdq {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero 2275; AVX512-NEXT: vmovq %xmm2, %rdx 2276; AVX512-NEXT: vpextrq $1, %xmm2, %r10 2277; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm2 2278; AVX512-NEXT: vmovq %xmm2, %r13 2279; AVX512-NEXT: vpextrq $1, %xmm2, %r14 2280; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm1 2281; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero 2282; AVX512-NEXT: vpmovzxdq {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero 2283; AVX512-NEXT: vmovq %xmm2, %r15 2284; AVX512-NEXT: vpextrq $1, %xmm2, %r8 2285; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm2 2286; AVX512-NEXT: vmovq %xmm2, %r11 2287; AVX512-NEXT: vpextrq $1, %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill 2288; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm1 2289; AVX512-NEXT: vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero 2290; AVX512-NEXT: vmovq %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill 2291; AVX512-NEXT: vpextrq $1, %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill 2292; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm1 2293; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero 2294; AVX512-NEXT: vpmovzxdq {{.*#+}} ymm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero 2295; AVX512-NEXT: vextracti128 $1, %ymm3, %xmm4 2296; AVX512-NEXT: vmovq %xmm4, %rax 2297; AVX512-NEXT: addq %rbx, %rax 2298; AVX512-NEXT: movq %rax, %rbx 2299; AVX512-NEXT: vpextrq $1, %xmm4, %rax 2300; AVX512-NEXT: addq %rbp, %rax 2301; AVX512-NEXT: movq %rax, %r9 2302; AVX512-NEXT: vmovq %xmm3, %rcx 2303; AVX512-NEXT: addq %rdi, %rcx 2304; AVX512-NEXT: vpextrq $1, %xmm3, %rax 2305; AVX512-NEXT: addq %rsi, %rax 2306; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2307; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm2 2308; AVX512-NEXT: vpmovzxdq {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero 2309; AVX512-NEXT: vmovq %xmm2, %rbp 2310; AVX512-NEXT: addq %rdx, %rbp 2311; AVX512-NEXT: vpextrq $1, %xmm2, %rax 2312; AVX512-NEXT: addq %r10, %rax 2313; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2314; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm2 2315; AVX512-NEXT: vmovq %xmm2, %rax 2316; AVX512-NEXT: addq %r13, %rax 2317; AVX512-NEXT: movq %rax, %r13 2318; AVX512-NEXT: vpextrq $1, %xmm2, %rax 2319; AVX512-NEXT: addq %r14, %rax 2320; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2321; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm0 2322; AVX512-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero 2323; AVX512-NEXT: vpmovzxdq {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero 2324; AVX512-NEXT: vmovq %xmm2, %rax 2325; AVX512-NEXT: addq %r15, %rax 2326; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2327; AVX512-NEXT: vpextrq $1, %xmm2, %rax 2328; AVX512-NEXT: addq %r8, %rax 2329; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2330; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm2 2331; AVX512-NEXT: vmovq %xmm2, %rax 2332; AVX512-NEXT: addq %r11, %rax 2333; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2334; AVX512-NEXT: vpextrq $1, %xmm2, %r12 2335; AVX512-NEXT: addq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Folded Reload 2336; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm0 2337; AVX512-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero 2338; AVX512-NEXT: vmovq %xmm0, %r11 2339; AVX512-NEXT: addq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Folded Reload 2340; AVX512-NEXT: vpextrq $1, %xmm0, %r8 2341; AVX512-NEXT: addq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Folded Reload 2342; AVX512-NEXT: vmovq %xmm1, %rax 2343; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm0 2344; AVX512-NEXT: vmovq %xmm0, %rdi 2345; AVX512-NEXT: addq %rax, %rdi 2346; AVX512-NEXT: vpextrq $1, %xmm1, %rsi 2347; AVX512-NEXT: vpextrq $1, %xmm0, %rdx 2348; AVX512-NEXT: addq %rsi, %rdx 2349; AVX512-NEXT: addq $-1, %rbx 2350; AVX512-NEXT: movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2351; AVX512-NEXT: movl $0, %r15d 2352; AVX512-NEXT: adcq $-1, %r15 2353; AVX512-NEXT: addq $-1, %r9 2354; AVX512-NEXT: movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2355; AVX512-NEXT: movl $0, %r14d 2356; AVX512-NEXT: adcq $-1, %r14 2357; AVX512-NEXT: addq $-1, %rcx 2358; AVX512-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2359; AVX512-NEXT: movl $0, %esi 2360; AVX512-NEXT: adcq $-1, %rsi 2361; AVX512-NEXT: addq $-1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill 2362; AVX512-NEXT: movl $0, %r10d 2363; AVX512-NEXT: adcq $-1, %r10 2364; AVX512-NEXT: addq $-1, %rbp 2365; AVX512-NEXT: movq %rbp, (%rsp) # 8-byte Spill 2366; AVX512-NEXT: movl $0, %r9d 2367; AVX512-NEXT: adcq $-1, %r9 2368; AVX512-NEXT: addq $-1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill 2369; AVX512-NEXT: movl $0, %eax 2370; AVX512-NEXT: adcq $-1, %rax 2371; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2372; AVX512-NEXT: addq $-1, %r13 2373; AVX512-NEXT: movq %r13, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2374; AVX512-NEXT: movl $0, %eax 2375; AVX512-NEXT: adcq $-1, %rax 2376; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2377; AVX512-NEXT: addq $-1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill 2378; AVX512-NEXT: movl $0, %eax 2379; AVX512-NEXT: adcq $-1, %rax 2380; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2381; AVX512-NEXT: addq $-1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill 2382; AVX512-NEXT: movl $0, %eax 2383; AVX512-NEXT: adcq $-1, %rax 2384; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2385; AVX512-NEXT: addq $-1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill 2386; AVX512-NEXT: movl $0, %eax 2387; AVX512-NEXT: adcq $-1, %rax 2388; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2389; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload 2390; AVX512-NEXT: addq $-1, %rcx 2391; AVX512-NEXT: movl $0, %eax 2392; AVX512-NEXT: adcq $-1, %rax 2393; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2394; AVX512-NEXT: addq $-1, %r12 2395; AVX512-NEXT: movl $0, %eax 2396; AVX512-NEXT: adcq $-1, %rax 2397; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2398; AVX512-NEXT: addq $-1, %r11 2399; AVX512-NEXT: movl $0, %r13d 2400; AVX512-NEXT: adcq $-1, %r13 2401; AVX512-NEXT: addq $-1, %r8 2402; AVX512-NEXT: movl $0, %eax 2403; AVX512-NEXT: adcq $-1, %rax 2404; AVX512-NEXT: addq $-1, %rdi 2405; AVX512-NEXT: movl $0, %ebx 2406; AVX512-NEXT: adcq $-1, %rbx 2407; AVX512-NEXT: addq $-1, %rdx 2408; AVX512-NEXT: movl $0, %ebp 2409; AVX512-NEXT: adcq $-1, %rbp 2410; AVX512-NEXT: shldq $63, %rdx, %rbp 2411; AVX512-NEXT: movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2412; AVX512-NEXT: shldq $63, %rdi, %rbx 2413; AVX512-NEXT: movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill 2414; AVX512-NEXT: shldq $63, %r8, %rax 2415; AVX512-NEXT: movq %rax, %r8 2416; AVX512-NEXT: shldq $63, %r11, %r13 2417; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload 2418; AVX512-NEXT: shldq $63, %r12, %r11 2419; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Reload 2420; AVX512-NEXT: shldq $63, %rcx, %r12 2421; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload 2422; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload 2423; AVX512-NEXT: shldq $63, %rax, %rbp 2424; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload 2425; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload 2426; AVX512-NEXT: shldq $63, %rax, %rbx 2427; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload 2428; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload 2429; AVX512-NEXT: shldq $63, %rax, %rdi 2430; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload 2431; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload 2432; AVX512-NEXT: shldq $63, %rax, %rdx 2433; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload 2434; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload 2435; AVX512-NEXT: shldq $63, %rax, %rcx 2436; AVX512-NEXT: movq (%rsp), %rax # 8-byte Reload 2437; AVX512-NEXT: shldq $63, %rax, %r9 2438; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload 2439; AVX512-NEXT: shldq $63, %rax, %r10 2440; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload 2441; AVX512-NEXT: shldq $63, %rax, %rsi 2442; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload 2443; AVX512-NEXT: shldq $63, %rax, %r14 2444; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload 2445; AVX512-NEXT: shldq $63, %rax, %r15 2446; AVX512-NEXT: vmovq %r15, %xmm0 2447; AVX512-NEXT: vmovq %r14, %xmm1 2448; AVX512-NEXT: vmovq %r10, %xmm2 2449; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 2450; AVX512-NEXT: vmovq %rsi, %xmm1 2451; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 2452; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 2453; AVX512-NEXT: vextracti32x4 $2, %zmm0, %xmm2 2454; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm1 2455; AVX512-NEXT: vmovd %esi, %xmm3 2456; AVX512-NEXT: vmovd %xmm1, %eax 2457; AVX512-NEXT: vpinsrb $1, %eax, %xmm3, %xmm1 2458; AVX512-NEXT: vmovd %xmm2, %eax 2459; AVX512-NEXT: vpinsrb $2, %eax, %xmm1, %xmm1 2460; AVX512-NEXT: vextracti32x4 $3, %zmm0, %xmm0 2461; AVX512-NEXT: vmovd %xmm0, %eax 2462; AVX512-NEXT: vpinsrb $3, %eax, %xmm1, %xmm0 2463; AVX512-NEXT: vpinsrb $4, %r9d, %xmm0, %xmm0 2464; AVX512-NEXT: vmovq %r9, %xmm1 2465; AVX512-NEXT: vmovq %rcx, %xmm2 2466; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 2467; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm2 2468; AVX512-NEXT: vmovd %xmm2, %eax 2469; AVX512-NEXT: vpinsrb $5, %eax, %xmm0, %xmm0 2470; AVX512-NEXT: vmovq %rdx, %xmm2 2471; AVX512-NEXT: vmovq %rdi, %xmm3 2472; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2 2473; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1 2474; AVX512-NEXT: vextracti32x4 $2, %zmm1, %xmm2 2475; AVX512-NEXT: vmovd %xmm2, %eax 2476; AVX512-NEXT: vpinsrb $6, %eax, %xmm0, %xmm0 2477; AVX512-NEXT: vextracti32x4 $3, %zmm1, %xmm1 2478; AVX512-NEXT: vmovd %xmm1, %eax 2479; AVX512-NEXT: vpinsrb $7, %eax, %xmm0, %xmm0 2480; AVX512-NEXT: movq %rbx, %rax 2481; AVX512-NEXT: vpinsrb $8, %eax, %xmm0, %xmm0 2482; AVX512-NEXT: vmovq %rbx, %xmm1 2483; AVX512-NEXT: vmovq %rbp, %xmm2 2484; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 2485; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm2 2486; AVX512-NEXT: vmovd %xmm2, %eax 2487; AVX512-NEXT: vpinsrb $9, %eax, %xmm0, %xmm0 2488; AVX512-NEXT: vmovq %r12, %xmm2 2489; AVX512-NEXT: vmovq %r11, %xmm3 2490; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2 2491; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1 2492; AVX512-NEXT: vextracti32x4 $2, %zmm1, %xmm2 2493; AVX512-NEXT: vmovd %xmm2, %eax 2494; AVX512-NEXT: vpinsrb $10, %eax, %xmm0, %xmm0 2495; AVX512-NEXT: vextracti32x4 $3, %zmm1, %xmm1 2496; AVX512-NEXT: vmovd %xmm1, %eax 2497; AVX512-NEXT: vpinsrb $11, %eax, %xmm0, %xmm0 2498; AVX512-NEXT: vpinsrb $12, %r13d, %xmm0, %xmm0 2499; AVX512-NEXT: vmovq %r13, %xmm1 2500; AVX512-NEXT: vmovq %r8, %xmm2 2501; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 2502; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm2 2503; AVX512-NEXT: vmovd %xmm2, %eax 2504; AVX512-NEXT: vpinsrb $13, %eax, %xmm0, %xmm0 2505; AVX512-NEXT: vmovq {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 8-byte Folded Reload 2506; AVX512-NEXT: # xmm2 = mem[0],zero 2507; AVX512-NEXT: vmovq {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 8-byte Folded Reload 2508; AVX512-NEXT: # xmm3 = mem[0],zero 2509; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2 2510; AVX512-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1 2511; AVX512-NEXT: vextracti32x4 $2, %zmm1, %xmm2 2512; AVX512-NEXT: vmovd %xmm2, %eax 2513; AVX512-NEXT: vpinsrb $14, %eax, %xmm0, %xmm0 2514; AVX512-NEXT: vextracti32x4 $3, %zmm1, %xmm1 2515; AVX512-NEXT: vmovd %xmm1, %eax 2516; AVX512-NEXT: vpinsrb $15, %eax, %xmm0, %xmm0 2517; AVX512-NEXT: vmovdqu %xmm0, (%rax) 2518; AVX512-NEXT: addq $24, %rsp 2519; AVX512-NEXT: popq %rbx 2520; AVX512-NEXT: popq %r12 2521; AVX512-NEXT: popq %r13 2522; AVX512-NEXT: popq %r14 2523; AVX512-NEXT: popq %r15 2524; AVX512-NEXT: popq %rbp 2525; AVX512-NEXT: vzeroupper 2526; AVX512-NEXT: retq 2527 %1 = load <16 x i8>, ptr %a 2528 %2 = load <16 x i8>, ptr %b 2529 %3 = zext <16 x i8> %1 to <16 x i128> 2530 %4 = zext <16 x i8> %2 to <16 x i128> 2531 %5 = add <16 x i128> %3, <i128 -1, i128 -1, i128 -1, i128 -1, i128 -1, i128 -1, i128 -1, i128 -1, i128 -1, i128 -1, i128 -1, i128 -1, i128 -1, i128 -1, i128 -1, i128 -1> 2532 %6 = add <16 x i128> %5, %4 2533 %7 = lshr <16 x i128> %6, <i128 1, i128 1, i128 1, i128 1, i128 1, i128 1, i128 1, i128 1, i128 1, i128 1, i128 1, i128 1, i128 1, i128 1, i128 1, i128 1> 2534 %8 = trunc <16 x i128> %7 to <16 x i8> 2535 store <16 x i8> %8, ptr undef, align 4 2536 ret void 2537} 2538 2539; Make sure we don't fail on single element vectors. 2540define <1 x i8> @avg_v1i8(<1 x i8> %x, <1 x i8> %y) { 2541; SSE2-LABEL: avg_v1i8: 2542; SSE2: # %bb.0: 2543; SSE2-NEXT: movzbl %dil, %eax 2544; SSE2-NEXT: movzbl %sil, %ecx 2545; SSE2-NEXT: leal 1(%rax,%rcx), %eax 2546; SSE2-NEXT: shrl %eax 2547; SSE2-NEXT: # kill: def $al killed $al killed $eax 2548; SSE2-NEXT: retq 2549; 2550; AVX-LABEL: avg_v1i8: 2551; AVX: # %bb.0: 2552; AVX-NEXT: movzbl %dil, %eax 2553; AVX-NEXT: movzbl %sil, %ecx 2554; AVX-NEXT: leal 1(%rax,%rcx), %eax 2555; AVX-NEXT: shrl %eax 2556; AVX-NEXT: # kill: def $al killed $al killed $eax 2557; AVX-NEXT: retq 2558 %a = zext <1 x i8> %x to <1 x i16> 2559 %b = zext <1 x i8> %y to <1 x i16> 2560 %c = add <1 x i16> %a, %b 2561 %d = add <1 x i16> %c, <i16 1> 2562 %e = lshr <1 x i16> %d, <i16 1> 2563 %f = trunc <1 x i16> %e to <1 x i8> 2564 ret <1 x i8> %f 2565} 2566 2567; _mm_avg_epu16( _mm_slli_epi16(a, 2), _mm_slli_epi16(b, 2)) 2568define <2 x i64> @PR41316(<2 x i64>, <2 x i64>) { 2569; SSE2-LABEL: PR41316: 2570; SSE2: # %bb.0: 2571; SSE2-NEXT: psllw $2, %xmm0 2572; SSE2-NEXT: psllw $2, %xmm1 2573; SSE2-NEXT: pavgw %xmm1, %xmm0 2574; SSE2-NEXT: retq 2575; 2576; AVX-LABEL: PR41316: 2577; AVX: # %bb.0: 2578; AVX-NEXT: vpsllw $2, %xmm0, %xmm0 2579; AVX-NEXT: vpsllw $2, %xmm1, %xmm1 2580; AVX-NEXT: vpavgw %xmm0, %xmm1, %xmm0 2581; AVX-NEXT: retq 2582 %3 = bitcast <2 x i64> %0 to <8 x i16> 2583 %4 = shl <8 x i16> %3, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2> 2584 %5 = bitcast <2 x i64> %1 to <8 x i16> 2585 %6 = shl <8 x i16> %5, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2> 2586 %7 = zext <8 x i16> %6 to <8 x i32> 2587 %8 = or <8 x i16> %4, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> 2588 %9 = zext <8 x i16> %8 to <8 x i32> 2589 %10 = add nuw nsw <8 x i32> %9, %7 2590 %11 = lshr <8 x i32> %10, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 2591 %12 = trunc <8 x i32> %11 to <8 x i16> 2592 %13 = bitcast <8 x i16> %12 to <2 x i64> 2593 ret <2 x i64> %13 2594} 2595 2596; shuffle(avg(shuffle(),shuffle())) -> avg(shuffle(),shuffle()) 2597define <16 x i8> @fold_avgb_shuffles(<16 x i8> %x, <16 x i8> %y) { 2598; SSE2-LABEL: fold_avgb_shuffles: 2599; SSE2: # %bb.0: # %entry 2600; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,0,2,2] 2601; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] 2602; SSE2-NEXT: pavgb %xmm1, %xmm0 2603; SSE2-NEXT: retq 2604; 2605; AVX-LABEL: fold_avgb_shuffles: 2606; AVX: # %bb.0: # %entry 2607; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,0,2,2] 2608; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] 2609; AVX-NEXT: vpavgb %xmm1, %xmm0, %xmm0 2610; AVX-NEXT: retq 2611entry: 2612 %0 = shufflevector <16 x i8> %x, <16 x i8> poison, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7> 2613 %1 = shufflevector <16 x i8> %y, <16 x i8> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 8, i32 9, i32 10, i32 11, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3> 2614 %2 = tail call <16 x i8> @llvm.x86.sse2.pavg.b(<16 x i8> %0, <16 x i8> %1) 2615 %3 = shufflevector <16 x i8> %2, <16 x i8> poison, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3> 2616 ret <16 x i8> %3 2617} 2618declare <16 x i8> @llvm.x86.sse2.pavg.b(<16 x i8>, <16 x i8>) 2619 2620define <8 x i16> @fold_avgw_shuffles(<8 x i16> %x, <8 x i16> %y) { 2621; SSE2-LABEL: fold_avgw_shuffles: 2622; SSE2: # %bb.0: # %entry 2623; SSE2-NEXT: pavgw %xmm1, %xmm0 2624; SSE2-NEXT: retq 2625; 2626; AVX-LABEL: fold_avgw_shuffles: 2627; AVX: # %bb.0: # %entry 2628; AVX-NEXT: vpavgw %xmm1, %xmm0, %xmm0 2629; AVX-NEXT: retq 2630entry: 2631 %0 = shufflevector <8 x i16> %x, <8 x i16> poison, <8 x i32> <i32 6, i32 7, i32 4, i32 5, i32 2, i32 3, i32 0, i32 1> 2632 %1 = shufflevector <8 x i16> %y, <8 x i16> poison, <8 x i32> <i32 6, i32 7, i32 4, i32 5, i32 2, i32 3, i32 0, i32 1> 2633 %2 = tail call <8 x i16> @llvm.x86.sse2.pavg.w(<8 x i16> %0, <8 x i16> %1) 2634 %3 = shufflevector <8 x i16> %2, <8 x i16> poison, <8 x i32> <i32 6, i32 7, i32 4, i32 5, i32 2, i32 3, i32 0, i32 1> 2635 ret <8 x i16> %3 2636} 2637declare <8 x i16> @llvm.x86.sse2.pavg.w(<8 x i16>, <8 x i16>) 2638 2639define <8 x i16> @PR52131_pavg_chain(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c) { 2640; SSE2-LABEL: PR52131_pavg_chain: 2641; SSE2: # %bb.0: 2642; SSE2-NEXT: pavgw %xmm1, %xmm0 2643; SSE2-NEXT: pavgw %xmm2, %xmm0 2644; SSE2-NEXT: retq 2645; 2646; AVX-LABEL: PR52131_pavg_chain: 2647; AVX: # %bb.0: 2648; AVX-NEXT: vpavgw %xmm1, %xmm0, %xmm0 2649; AVX-NEXT: vpavgw %xmm0, %xmm2, %xmm0 2650; AVX-NEXT: retq 2651 %i = zext <8 x i16> %a to <8 x i32> 2652 %i1 = zext <8 x i16> %b to <8 x i32> 2653 %i2 = add nuw nsw <8 x i32> %i, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 2654 %i3 = add nuw nsw <8 x i32> %i2, %i1 2655 %i4 = lshr <8 x i32> %i3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 2656 %i5 = and <8 x i32> %i4, <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535> 2657 %i6 = zext <8 x i16> %c to <8 x i32> 2658 %i7 = add nuw nsw <8 x i32> %i6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 2659 %i8 = add nuw nsw <8 x i32> %i7, %i5 2660 %i9 = lshr <8 x i32> %i8, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 2661 %i10 = trunc <8 x i32> %i9 to <8 x i16> 2662 ret <8 x i16> %i10 2663} 2664 2665define <8 x i16> @PR52131_pavg_chainlike_but_not_zext(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c) { 2666; SSE2-LABEL: PR52131_pavg_chainlike_but_not_zext: 2667; SSE2: # %bb.0: 2668; SSE2-NEXT: pavgw %xmm1, %xmm0 2669; SSE2-NEXT: pavgw %xmm2, %xmm0 2670; SSE2-NEXT: retq 2671; 2672; AVX-LABEL: PR52131_pavg_chainlike_but_not_zext: 2673; AVX: # %bb.0: 2674; AVX-NEXT: vpavgw %xmm1, %xmm0, %xmm0 2675; AVX-NEXT: vpavgw %xmm0, %xmm2, %xmm0 2676; AVX-NEXT: retq 2677 %i = zext <8 x i16> %a to <8 x i32> 2678 %i1 = zext <8 x i16> %b to <8 x i32> 2679 %i2 = add nuw nsw <8 x i32> %i, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 2680 %i3 = add nuw nsw <8 x i32> %i2, %i1 2681 %i4 = lshr <8 x i32> %i3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 2682 %i5 = and <8 x i32> %i4, <i32 131071, i32 131071, i32 131071, i32 131071, i32 131071, i32 131071, i32 131071, i32 131071> 2683 %i6 = zext <8 x i16> %c to <8 x i32> 2684 %i7 = add nuw nsw <8 x i32> %i6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 2685 %i8 = add nuw nsw <8 x i32> %i7, %i5 2686 %i9 = lshr <8 x i32> %i8, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 2687 %i10 = trunc <8 x i32> %i9 to <8 x i16> 2688 ret <8 x i16> %i10 2689} 2690 2691define <8 x i16> @PR52131_pavg_with_mask(<8 x i32> %a, <8 x i16> %b) { 2692; SSE2-LABEL: PR52131_pavg_with_mask: 2693; SSE2: # %bb.0: 2694; SSE2-NEXT: pslld $16, %xmm1 2695; SSE2-NEXT: psrad $16, %xmm1 2696; SSE2-NEXT: pslld $16, %xmm0 2697; SSE2-NEXT: psrad $16, %xmm0 2698; SSE2-NEXT: packssdw %xmm1, %xmm0 2699; SSE2-NEXT: pavgw %xmm2, %xmm0 2700; SSE2-NEXT: retq 2701; 2702; AVX1-LABEL: PR52131_pavg_with_mask: 2703; AVX1: # %bb.0: 2704; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 2705; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 2706; AVX1-NEXT: vpackusdw %xmm2, %xmm0, %xmm0 2707; AVX1-NEXT: vpavgw %xmm0, %xmm1, %xmm0 2708; AVX1-NEXT: vzeroupper 2709; AVX1-NEXT: retq 2710; 2711; AVX2-LABEL: PR52131_pavg_with_mask: 2712; AVX2: # %bb.0: 2713; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u] 2714; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3] 2715; AVX2-NEXT: vpavgw %xmm0, %xmm1, %xmm0 2716; AVX2-NEXT: vzeroupper 2717; AVX2-NEXT: retq 2718; 2719; AVX512-LABEL: PR52131_pavg_with_mask: 2720; AVX512: # %bb.0: 2721; AVX512-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0 2722; AVX512-NEXT: vpmovdw %zmm0, %ymm0 2723; AVX512-NEXT: vpavgw %xmm0, %xmm1, %xmm0 2724; AVX512-NEXT: vzeroupper 2725; AVX512-NEXT: retq 2726 %i = and <8 x i32> %a, <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535> 2727 %i3 = zext <8 x i16> %b to <8 x i32> 2728 %i4 = add nuw nsw <8 x i32> %i3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 2729 %i5 = add nuw nsw <8 x i32> %i4, %i 2730 %i6 = lshr <8 x i32> %i5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 2731 %i7 = trunc <8 x i32> %i6 to <8 x i16> 2732 ret <8 x i16> %i7 2733} 2734 2735define <8 x i16> @PR52131_not_zext_with_constant(<8 x i32> %a) { 2736; SSE2-LABEL: PR52131_not_zext_with_constant: 2737; SSE2: # %bb.0: 2738; SSE2-NEXT: pslld $16, %xmm1 2739; SSE2-NEXT: psrad $16, %xmm1 2740; SSE2-NEXT: pslld $16, %xmm0 2741; SSE2-NEXT: psrad $16, %xmm0 2742; SSE2-NEXT: packssdw %xmm1, %xmm0 2743; SSE2-NEXT: pavgw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 2744; SSE2-NEXT: retq 2745; 2746; AVX1-LABEL: PR52131_not_zext_with_constant: 2747; AVX1: # %bb.0: 2748; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 2749; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 2750; AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0 2751; AVX1-NEXT: vpavgw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 2752; AVX1-NEXT: vzeroupper 2753; AVX1-NEXT: retq 2754; 2755; AVX2-LABEL: PR52131_not_zext_with_constant: 2756; AVX2: # %bb.0: 2757; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u] 2758; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3] 2759; AVX2-NEXT: vpavgw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 2760; AVX2-NEXT: vzeroupper 2761; AVX2-NEXT: retq 2762; 2763; AVX512-LABEL: PR52131_not_zext_with_constant: 2764; AVX512: # %bb.0: 2765; AVX512-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0 2766; AVX512-NEXT: vpmovdw %zmm0, %ymm0 2767; AVX512-NEXT: vpavgw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 2768; AVX512-NEXT: vzeroupper 2769; AVX512-NEXT: retq 2770 %i = and <8 x i32> %a, <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535> 2771 %i1 = add nuw nsw <8 x i32> %i, <i32 43, i32 43, i32 43, i32 43, i32 43, i32 43, i32 43, i32 43> 2772 %i2 = lshr <8 x i32> %i1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1> 2773 %i3 = trunc <8 x i32> %i2 to <8 x i16> 2774 ret <8 x i16> %i3 2775} 2776