Lines Matching refs:Intrinsic
42 static bool UpgradePTESTIntrinsic(Function* F, Intrinsic::ID IID, in UpgradePTESTIntrinsic()
52 NewFn = Intrinsic::getDeclaration(F->getParent(), IID); in UpgradePTESTIntrinsic()
58 static bool UpgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, in UpgradeX86IntrinsicsWith8BitMask()
68 NewFn = Intrinsic::getDeclaration(F->getParent(), IID); in UpgradeX86IntrinsicsWith8BitMask()
74 static bool UpgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID, in UpgradeX86MaskedFPCompare()
81 NewFn = Intrinsic::getDeclaration(F->getParent(), IID); in UpgradeX86MaskedFPCompare()
439 NewFn = Intrinsic::getDeclaration(F->getParent(), in UpgradeX86IntrinsicFunction()
440 Intrinsic::x86_rdtscp); in UpgradeX86IntrinsicFunction()
447 return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestc, NewFn); in UpgradeX86IntrinsicFunction()
449 return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestz, NewFn); in UpgradeX86IntrinsicFunction()
451 return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestnzc, NewFn); in UpgradeX86IntrinsicFunction()
456 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_insertps, in UpgradeX86IntrinsicFunction()
459 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dppd, in UpgradeX86IntrinsicFunction()
462 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dpps, in UpgradeX86IntrinsicFunction()
465 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_mpsadbw, in UpgradeX86IntrinsicFunction()
468 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx_dp_ps_256, in UpgradeX86IntrinsicFunction()
471 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_mpsadbw, in UpgradeX86IntrinsicFunction()
474 return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_128, in UpgradeX86IntrinsicFunction()
477 return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_256, in UpgradeX86IntrinsicFunction()
480 return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_512, in UpgradeX86IntrinsicFunction()
483 return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_128, in UpgradeX86IntrinsicFunction()
486 return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_256, in UpgradeX86IntrinsicFunction()
489 return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_512, in UpgradeX86IntrinsicFunction()
495 NewFn = Intrinsic::getDeclaration(F->getParent(), in UpgradeX86IntrinsicFunction()
496 Intrinsic::x86_xop_vfrcz_ss); in UpgradeX86IntrinsicFunction()
501 NewFn = Intrinsic::getDeclaration(F->getParent(), in UpgradeX86IntrinsicFunction()
502 Intrinsic::x86_xop_vfrcz_sd); in UpgradeX86IntrinsicFunction()
512 Intrinsic::ID Permil2ID; in UpgradeX86IntrinsicFunction()
514 Permil2ID = Intrinsic::x86_xop_vpermil2pd; in UpgradeX86IntrinsicFunction()
516 Permil2ID = Intrinsic::x86_xop_vpermil2ps; in UpgradeX86IntrinsicFunction()
518 Permil2ID = Intrinsic::x86_xop_vpermil2pd_256; in UpgradeX86IntrinsicFunction()
520 Permil2ID = Intrinsic::x86_xop_vpermil2ps_256; in UpgradeX86IntrinsicFunction()
521 NewFn = Intrinsic::getDeclaration(F->getParent(), Permil2ID); in UpgradeX86IntrinsicFunction()
527 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::eh_recoverfp); in UpgradeX86IntrinsicFunction()
547 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse, in UpgradeIntrinsicFunction1()
552 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::roundeven, in UpgradeIntrinsicFunction1()
557 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse, in UpgradeIntrinsicFunction1()
575 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop, in UpgradeIntrinsicFunction1()
581 static const Intrinsic::ID StoreInts[] = {Intrinsic::arm_neon_vst1, in UpgradeIntrinsicFunction1()
582 Intrinsic::arm_neon_vst2, in UpgradeIntrinsicFunction1()
583 Intrinsic::arm_neon_vst3, in UpgradeIntrinsicFunction1()
584 Intrinsic::arm_neon_vst4}; in UpgradeIntrinsicFunction1()
586 static const Intrinsic::ID StoreLaneInts[] = { in UpgradeIntrinsicFunction1()
587 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane, in UpgradeIntrinsicFunction1()
588 Intrinsic::arm_neon_vst4lane in UpgradeIntrinsicFunction1()
594 NewFn = Intrinsic::getDeclaration(F->getParent(), in UpgradeIntrinsicFunction1()
597 NewFn = Intrinsic::getDeclaration(F->getParent(), in UpgradeIntrinsicFunction1()
602 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::thread_pointer); in UpgradeIntrinsicFunction1()
606 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::sadd_sat, in UpgradeIntrinsicFunction1()
611 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::uadd_sat, in UpgradeIntrinsicFunction1()
616 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ssub_sat, in UpgradeIntrinsicFunction1()
621 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::usub_sat, in UpgradeIntrinsicFunction1()
630 NewFn = Intrinsic::getDeclaration(F->getParent(), in UpgradeIntrinsicFunction1()
631 Intrinsic::aarch64_neon_faddp, Ty); in UpgradeIntrinsicFunction1()
641 Intrinsic::ID IID = in UpgradeIntrinsicFunction1()
642 StringSwitch<Intrinsic::ID>(Name) in UpgradeIntrinsicFunction1()
645 Intrinsic::arm_neon_bfdot) in UpgradeIntrinsicFunction1()
648 Intrinsic::aarch64_neon_bfdot) in UpgradeIntrinsicFunction1()
649 .Default(Intrinsic::not_intrinsic); in UpgradeIntrinsicFunction1()
650 if (IID == Intrinsic::not_intrinsic) in UpgradeIntrinsicFunction1()
661 NewFn = Intrinsic::getDeclaration(F->getParent(), IID, Tys); in UpgradeIntrinsicFunction1()
670 Intrinsic::ID IID = in UpgradeIntrinsicFunction1()
671 StringSwitch<Intrinsic::ID>(Name) in UpgradeIntrinsicFunction1()
673 Intrinsic::arm_neon_bfmmla) in UpgradeIntrinsicFunction1()
675 Intrinsic::arm_neon_bfmlalb) in UpgradeIntrinsicFunction1()
677 Intrinsic::arm_neon_bfmlalt) in UpgradeIntrinsicFunction1()
679 Intrinsic::aarch64_neon_bfmmla) in UpgradeIntrinsicFunction1()
681 Intrinsic::aarch64_neon_bfmlalb) in UpgradeIntrinsicFunction1()
683 Intrinsic::aarch64_neon_bfmlalt) in UpgradeIntrinsicFunction1()
684 .Default(Intrinsic::not_intrinsic); in UpgradeIntrinsicFunction1()
685 if (IID == Intrinsic::not_intrinsic) in UpgradeIntrinsicFunction1()
689 NewFn = Intrinsic::getDeclaration(F->getParent(), IID, Tys); in UpgradeIntrinsicFunction1()
719 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::fshr, in UpgradeIntrinsicFunction1()
730 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz, in UpgradeIntrinsicFunction1()
736 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz, in UpgradeIntrinsicFunction1()
745 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value); in UpgradeIntrinsicFunction1()
754 NewFn = Intrinsic::getDeclaration(F->getParent(), in UpgradeIntrinsicFunction1()
755 Intrinsic::vector_extract, Tys); in UpgradeIntrinsicFunction1()
763 NewFn = Intrinsic::getDeclaration(F->getParent(), in UpgradeIntrinsicFunction1()
764 Intrinsic::vector_insert, Tys); in UpgradeIntrinsicFunction1()
771 Intrinsic::ID ID; in UpgradeIntrinsicFunction1()
772 ID = StringSwitch<Intrinsic::ID>(Groups[1]) in UpgradeIntrinsicFunction1()
773 .Case("add", Intrinsic::vector_reduce_add) in UpgradeIntrinsicFunction1()
774 .Case("mul", Intrinsic::vector_reduce_mul) in UpgradeIntrinsicFunction1()
775 .Case("and", Intrinsic::vector_reduce_and) in UpgradeIntrinsicFunction1()
776 .Case("or", Intrinsic::vector_reduce_or) in UpgradeIntrinsicFunction1()
777 .Case("xor", Intrinsic::vector_reduce_xor) in UpgradeIntrinsicFunction1()
778 .Case("smax", Intrinsic::vector_reduce_smax) in UpgradeIntrinsicFunction1()
779 .Case("smin", Intrinsic::vector_reduce_smin) in UpgradeIntrinsicFunction1()
780 .Case("umax", Intrinsic::vector_reduce_umax) in UpgradeIntrinsicFunction1()
781 .Case("umin", Intrinsic::vector_reduce_umin) in UpgradeIntrinsicFunction1()
782 .Case("fmax", Intrinsic::vector_reduce_fmax) in UpgradeIntrinsicFunction1()
783 .Case("fmin", Intrinsic::vector_reduce_fmin) in UpgradeIntrinsicFunction1()
784 .Default(Intrinsic::not_intrinsic); in UpgradeIntrinsicFunction1()
785 if (ID != Intrinsic::not_intrinsic) { in UpgradeIntrinsicFunction1()
788 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, {Args[0]}); in UpgradeIntrinsicFunction1()
796 Intrinsic::ID ID = Intrinsic::not_intrinsic; in UpgradeIntrinsicFunction1()
798 ID = Intrinsic::vector_reduce_fadd; in UpgradeIntrinsicFunction1()
800 ID = Intrinsic::vector_reduce_fmul; in UpgradeIntrinsicFunction1()
801 if (ID != Intrinsic::not_intrinsic) { in UpgradeIntrinsicFunction1()
805 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys); in UpgradeIntrinsicFunction1()
815 Intrinsic::ID ID = IsLifetimeStart ? in UpgradeIntrinsicFunction1()
816 Intrinsic::lifetime_start : Intrinsic::invariant_start; in UpgradeIntrinsicFunction1()
819 if (F->getName() != Intrinsic::getName(ID, ObjectPtr, F->getParent())) { in UpgradeIntrinsicFunction1()
821 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ObjectPtr); in UpgradeIntrinsicFunction1()
828 Intrinsic::ID ID = IsLifetimeEnd ? in UpgradeIntrinsicFunction1()
829 Intrinsic::lifetime_end : Intrinsic::invariant_end; in UpgradeIntrinsicFunction1()
833 if (F->getName() != Intrinsic::getName(ID, ObjectPtr, F->getParent())) { in UpgradeIntrinsicFunction1()
835 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ObjectPtr); in UpgradeIntrinsicFunction1()
844 NewFn = Intrinsic::getDeclaration(F->getParent(), in UpgradeIntrinsicFunction1()
845 Intrinsic::launder_invariant_group, ObjectPtr); in UpgradeIntrinsicFunction1()
856 Intrinsic::getName(Intrinsic::masked_load, Tys, F->getParent())) { in UpgradeIntrinsicFunction1()
858 NewFn = Intrinsic::getDeclaration(F->getParent(), in UpgradeIntrinsicFunction1()
859 Intrinsic::masked_load, in UpgradeIntrinsicFunction1()
868 Intrinsic::getName(Intrinsic::masked_store, Tys, F->getParent())) { in UpgradeIntrinsicFunction1()
870 NewFn = Intrinsic::getDeclaration(F->getParent(), in UpgradeIntrinsicFunction1()
871 Intrinsic::masked_store, in UpgradeIntrinsicFunction1()
881 Intrinsic::getName(Intrinsic::masked_gather, Tys, F->getParent())) { in UpgradeIntrinsicFunction1()
883 NewFn = Intrinsic::getDeclaration(F->getParent(), in UpgradeIntrinsicFunction1()
884 Intrinsic::masked_gather, Tys); in UpgradeIntrinsicFunction1()
892 Intrinsic::getName(Intrinsic::masked_scatter, Tys, F->getParent())) { in UpgradeIntrinsicFunction1()
894 NewFn = Intrinsic::getDeclaration(F->getParent(), in UpgradeIntrinsicFunction1()
895 Intrinsic::masked_scatter, Tys); in UpgradeIntrinsicFunction1()
906 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memcpy, in UpgradeIntrinsicFunction1()
914 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memmove, in UpgradeIntrinsicFunction1()
926 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memset, in UpgradeIntrinsicFunction1()
937 Intrinsic::ID IID = StringSwitch<Intrinsic::ID>(Name) in UpgradeIntrinsicFunction1()
938 .Cases("brev32", "brev64", Intrinsic::bitreverse) in UpgradeIntrinsicFunction1()
939 .Case("clz.i", Intrinsic::ctlz) in UpgradeIntrinsicFunction1()
940 .Case("popc.i", Intrinsic::ctpop) in UpgradeIntrinsicFunction1()
941 .Default(Intrinsic::not_intrinsic); in UpgradeIntrinsicFunction1()
942 if (IID != Intrinsic::not_intrinsic && F->arg_size() == 1) { in UpgradeIntrinsicFunction1()
943 NewFn = Intrinsic::getDeclaration(F->getParent(), IID, in UpgradeIntrinsicFunction1()
974 Intrinsic::getName(Intrinsic::objectsize, Tys, F->getParent())) { in UpgradeIntrinsicFunction1()
976 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::objectsize, in UpgradeIntrinsicFunction1()
988 Intrinsic::getName(Intrinsic::prefetch, Tys, F->getParent())) { in UpgradeIntrinsicFunction1()
991 Intrinsic::getDeclaration(F->getParent(), Intrinsic::prefetch, Tys); in UpgradeIntrinsicFunction1()
996 NewFn = Intrinsic::getDeclaration(F->getParent(), in UpgradeIntrinsicFunction1()
997 Intrinsic::ptr_annotation, in UpgradeIntrinsicFunction1()
1013 NewFn = Intrinsic::getDeclaration(F->getParent(), in UpgradeIntrinsicFunction1()
1014 Intrinsic::var_annotation); in UpgradeIntrinsicFunction1()
1031 SmallVector<Intrinsic::IITDescriptor> Desc; in UpgradeIntrinsicFunction1()
1032 Intrinsic::getIntrinsicInfoTableEntries(F->getIntrinsicID(), Desc); in UpgradeIntrinsicFunction1()
1033 if (Desc.front().Kind == Intrinsic::IITDescriptor::Struct) { in UpgradeIntrinsicFunction1()
1043 if (auto Result = llvm::Intrinsic::remangleIntrinsicFunction(NewFn)) in UpgradeIntrinsicFunction1()
1050 auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F); in UpgradeIntrinsicFunction1()
1071 if (Intrinsic::ID id = F->getIntrinsicID()) in UpgradeIntrinsicFunction()
1072 F->setAttributes(Intrinsic::getAttributes(F->getContext(), id)); in UpgradeIntrinsicFunction()
1276 Intrinsic::ID IID; in UpgradeX86VPERMT2Intrinsics()
1278 IID = Intrinsic::x86_avx512_vpermi2var_ps_128; in UpgradeX86VPERMT2Intrinsics()
1280 IID = Intrinsic::x86_avx512_vpermi2var_d_128; in UpgradeX86VPERMT2Intrinsics()
1282 IID = Intrinsic::x86_avx512_vpermi2var_pd_128; in UpgradeX86VPERMT2Intrinsics()
1284 IID = Intrinsic::x86_avx512_vpermi2var_q_128; in UpgradeX86VPERMT2Intrinsics()
1286 IID = Intrinsic::x86_avx512_vpermi2var_ps_256; in UpgradeX86VPERMT2Intrinsics()
1288 IID = Intrinsic::x86_avx512_vpermi2var_d_256; in UpgradeX86VPERMT2Intrinsics()
1290 IID = Intrinsic::x86_avx512_vpermi2var_pd_256; in UpgradeX86VPERMT2Intrinsics()
1292 IID = Intrinsic::x86_avx512_vpermi2var_q_256; in UpgradeX86VPERMT2Intrinsics()
1294 IID = Intrinsic::x86_avx512_vpermi2var_ps_512; in UpgradeX86VPERMT2Intrinsics()
1296 IID = Intrinsic::x86_avx512_vpermi2var_d_512; in UpgradeX86VPERMT2Intrinsics()
1298 IID = Intrinsic::x86_avx512_vpermi2var_pd_512; in UpgradeX86VPERMT2Intrinsics()
1300 IID = Intrinsic::x86_avx512_vpermi2var_q_512; in UpgradeX86VPERMT2Intrinsics()
1302 IID = Intrinsic::x86_avx512_vpermi2var_hi_128; in UpgradeX86VPERMT2Intrinsics()
1304 IID = Intrinsic::x86_avx512_vpermi2var_hi_256; in UpgradeX86VPERMT2Intrinsics()
1306 IID = Intrinsic::x86_avx512_vpermi2var_hi_512; in UpgradeX86VPERMT2Intrinsics()
1308 IID = Intrinsic::x86_avx512_vpermi2var_qi_128; in UpgradeX86VPERMT2Intrinsics()
1310 IID = Intrinsic::x86_avx512_vpermi2var_qi_256; in UpgradeX86VPERMT2Intrinsics()
1312 IID = Intrinsic::x86_avx512_vpermi2var_qi_512; in UpgradeX86VPERMT2Intrinsics()
1323 Value *V = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID), in UpgradeX86VPERMT2Intrinsics()
1332 Intrinsic::ID IID) { in UpgradeX86BinaryIntrinsics()
1336 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); in UpgradeX86BinaryIntrinsics()
1362 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl; in upgradeX86Rotate()
1363 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); in upgradeX86Rotate()
1432 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl; in upgradeX86ConcatShift()
1433 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); in upgradeX86ConcatShift()
1495 Function *F = Intrinsic::getDeclaration(CI.getModule(), Intrinsic::abs, Ty); in upgradeAbs()
1587 Intrinsic::ID IID) { in UpgradeX86MaskedShift()
1588 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID); in UpgradeX86MaskedShift()
1624 Intrinsic::ID IID; in upgradeAVX512MaskToSelect()
1627 IID = Intrinsic::x86_sse_max_ps; in upgradeAVX512MaskToSelect()
1629 IID = Intrinsic::x86_sse2_max_pd; in upgradeAVX512MaskToSelect()
1631 IID = Intrinsic::x86_avx_max_ps_256; in upgradeAVX512MaskToSelect()
1633 IID = Intrinsic::x86_avx_max_pd_256; in upgradeAVX512MaskToSelect()
1638 IID = Intrinsic::x86_sse_min_ps; in upgradeAVX512MaskToSelect()
1640 IID = Intrinsic::x86_sse2_min_pd; in upgradeAVX512MaskToSelect()
1642 IID = Intrinsic::x86_avx_min_ps_256; in upgradeAVX512MaskToSelect()
1644 IID = Intrinsic::x86_avx_min_pd_256; in upgradeAVX512MaskToSelect()
1649 IID = Intrinsic::x86_ssse3_pshuf_b_128; in upgradeAVX512MaskToSelect()
1651 IID = Intrinsic::x86_avx2_pshuf_b; in upgradeAVX512MaskToSelect()
1653 IID = Intrinsic::x86_avx512_pshuf_b_512; in upgradeAVX512MaskToSelect()
1658 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128; in upgradeAVX512MaskToSelect()
1660 IID = Intrinsic::x86_avx2_pmul_hr_sw; in upgradeAVX512MaskToSelect()
1662 IID = Intrinsic::x86_avx512_pmul_hr_sw_512; in upgradeAVX512MaskToSelect()
1667 IID = Intrinsic::x86_sse2_pmulh_w; in upgradeAVX512MaskToSelect()
1669 IID = Intrinsic::x86_avx2_pmulh_w; in upgradeAVX512MaskToSelect()
1671 IID = Intrinsic::x86_avx512_pmulh_w_512; in upgradeAVX512MaskToSelect()
1676 IID = Intrinsic::x86_sse2_pmulhu_w; in upgradeAVX512MaskToSelect()
1678 IID = Intrinsic::x86_avx2_pmulhu_w; in upgradeAVX512MaskToSelect()
1680 IID = Intrinsic::x86_avx512_pmulhu_w_512; in upgradeAVX512MaskToSelect()
1685 IID = Intrinsic::x86_sse2_pmadd_wd; in upgradeAVX512MaskToSelect()
1687 IID = Intrinsic::x86_avx2_pmadd_wd; in upgradeAVX512MaskToSelect()
1689 IID = Intrinsic::x86_avx512_pmaddw_d_512; in upgradeAVX512MaskToSelect()
1694 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128; in upgradeAVX512MaskToSelect()
1696 IID = Intrinsic::x86_avx2_pmadd_ub_sw; in upgradeAVX512MaskToSelect()
1698 IID = Intrinsic::x86_avx512_pmaddubs_w_512; in upgradeAVX512MaskToSelect()
1703 IID = Intrinsic::x86_sse2_packsswb_128; in upgradeAVX512MaskToSelect()
1705 IID = Intrinsic::x86_avx2_packsswb; in upgradeAVX512MaskToSelect()
1707 IID = Intrinsic::x86_avx512_packsswb_512; in upgradeAVX512MaskToSelect()
1712 IID = Intrinsic::x86_sse2_packssdw_128; in upgradeAVX512MaskToSelect()
1714 IID = Intrinsic::x86_avx2_packssdw; in upgradeAVX512MaskToSelect()
1716 IID = Intrinsic::x86_avx512_packssdw_512; in upgradeAVX512MaskToSelect()
1721 IID = Intrinsic::x86_sse2_packuswb_128; in upgradeAVX512MaskToSelect()
1723 IID = Intrinsic::x86_avx2_packuswb; in upgradeAVX512MaskToSelect()
1725 IID = Intrinsic::x86_avx512_packuswb_512; in upgradeAVX512MaskToSelect()
1730 IID = Intrinsic::x86_sse41_packusdw; in upgradeAVX512MaskToSelect()
1732 IID = Intrinsic::x86_avx2_packusdw; in upgradeAVX512MaskToSelect()
1734 IID = Intrinsic::x86_avx512_packusdw_512; in upgradeAVX512MaskToSelect()
1739 IID = Intrinsic::x86_avx_vpermilvar_ps; in upgradeAVX512MaskToSelect()
1741 IID = Intrinsic::x86_avx_vpermilvar_pd; in upgradeAVX512MaskToSelect()
1743 IID = Intrinsic::x86_avx_vpermilvar_ps_256; in upgradeAVX512MaskToSelect()
1745 IID = Intrinsic::x86_avx_vpermilvar_pd_256; in upgradeAVX512MaskToSelect()
1747 IID = Intrinsic::x86_avx512_vpermilvar_ps_512; in upgradeAVX512MaskToSelect()
1749 IID = Intrinsic::x86_avx512_vpermilvar_pd_512; in upgradeAVX512MaskToSelect()
1753 IID = Intrinsic::x86_avx_cvt_pd2dq_256; in upgradeAVX512MaskToSelect()
1755 IID = Intrinsic::x86_avx_cvt_pd2_ps_256; in upgradeAVX512MaskToSelect()
1757 IID = Intrinsic::x86_avx_cvtt_pd2dq_256; in upgradeAVX512MaskToSelect()
1759 IID = Intrinsic::x86_sse2_cvttps2dq; in upgradeAVX512MaskToSelect()
1761 IID = Intrinsic::x86_avx_cvtt_ps2dq_256; in upgradeAVX512MaskToSelect()
1765 IID = Intrinsic::x86_avx2_permps; in upgradeAVX512MaskToSelect()
1767 IID = Intrinsic::x86_avx2_permd; in upgradeAVX512MaskToSelect()
1769 IID = Intrinsic::x86_avx512_permvar_df_256; in upgradeAVX512MaskToSelect()
1771 IID = Intrinsic::x86_avx512_permvar_di_256; in upgradeAVX512MaskToSelect()
1773 IID = Intrinsic::x86_avx512_permvar_sf_512; in upgradeAVX512MaskToSelect()
1775 IID = Intrinsic::x86_avx512_permvar_si_512; in upgradeAVX512MaskToSelect()
1777 IID = Intrinsic::x86_avx512_permvar_df_512; in upgradeAVX512MaskToSelect()
1779 IID = Intrinsic::x86_avx512_permvar_di_512; in upgradeAVX512MaskToSelect()
1781 IID = Intrinsic::x86_avx512_permvar_hi_128; in upgradeAVX512MaskToSelect()
1783 IID = Intrinsic::x86_avx512_permvar_hi_256; in upgradeAVX512MaskToSelect()
1785 IID = Intrinsic::x86_avx512_permvar_hi_512; in upgradeAVX512MaskToSelect()
1787 IID = Intrinsic::x86_avx512_permvar_qi_128; in upgradeAVX512MaskToSelect()
1789 IID = Intrinsic::x86_avx512_permvar_qi_256; in upgradeAVX512MaskToSelect()
1791 IID = Intrinsic::x86_avx512_permvar_qi_512; in upgradeAVX512MaskToSelect()
1796 IID = Intrinsic::x86_avx512_dbpsadbw_128; in upgradeAVX512MaskToSelect()
1798 IID = Intrinsic::x86_avx512_dbpsadbw_256; in upgradeAVX512MaskToSelect()
1800 IID = Intrinsic::x86_avx512_dbpsadbw_512; in upgradeAVX512MaskToSelect()
1805 IID = Intrinsic::x86_avx512_pmultishift_qb_128; in upgradeAVX512MaskToSelect()
1807 IID = Intrinsic::x86_avx512_pmultishift_qb_256; in upgradeAVX512MaskToSelect()
1809 IID = Intrinsic::x86_avx512_pmultishift_qb_512; in upgradeAVX512MaskToSelect()
1814 IID = Intrinsic::x86_avx512_conflict_d_128; in upgradeAVX512MaskToSelect()
1816 IID = Intrinsic::x86_avx512_conflict_d_256; in upgradeAVX512MaskToSelect()
1818 IID = Intrinsic::x86_avx512_conflict_d_512; in upgradeAVX512MaskToSelect()
1820 IID = Intrinsic::x86_avx512_conflict_q_128; in upgradeAVX512MaskToSelect()
1822 IID = Intrinsic::x86_avx512_conflict_q_256; in upgradeAVX512MaskToSelect()
1824 IID = Intrinsic::x86_avx512_conflict_q_512; in upgradeAVX512MaskToSelect()
1829 IID = Intrinsic::x86_sse2_pavg_b; in upgradeAVX512MaskToSelect()
1831 IID = Intrinsic::x86_avx2_pavg_b; in upgradeAVX512MaskToSelect()
1833 IID = Intrinsic::x86_avx512_pavg_b_512; in upgradeAVX512MaskToSelect()
1835 IID = Intrinsic::x86_sse2_pavg_w; in upgradeAVX512MaskToSelect()
1837 IID = Intrinsic::x86_avx2_pavg_w; in upgradeAVX512MaskToSelect()
1839 IID = Intrinsic::x86_avx512_pavg_w_512; in upgradeAVX512MaskToSelect()
1848 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID), in upgradeAVX512MaskToSelect()
1873 Intrinsic::getDeclaration(F->getParent(), Intrinsic::arm_mve_vctp64), in UpgradeARMIntrinsicCall()
1876 Intrinsic::getDeclaration( in UpgradeARMIntrinsicCall()
1877 F->getParent(), Intrinsic::arm_mve_pred_v2i, in UpgradeARMIntrinsicCall()
1881 Intrinsic::getDeclaration( in UpgradeARMIntrinsicCall()
1882 F->getParent(), Intrinsic::arm_mve_pred_i2v, in UpgradeARMIntrinsicCall()
1903 case Intrinsic::arm_mve_mull_int_predicated: in UpgradeARMIntrinsicCall()
1904 case Intrinsic::arm_mve_vqdmull_predicated: in UpgradeARMIntrinsicCall()
1905 case Intrinsic::arm_mve_vldr_gather_base_predicated: in UpgradeARMIntrinsicCall()
1908 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated: in UpgradeARMIntrinsicCall()
1909 case Intrinsic::arm_mve_vstr_scatter_base_predicated: in UpgradeARMIntrinsicCall()
1910 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated: in UpgradeARMIntrinsicCall()
1914 case Intrinsic::arm_mve_vldr_gather_offset_predicated: in UpgradeARMIntrinsicCall()
1918 case Intrinsic::arm_mve_vstr_scatter_offset_predicated: in UpgradeARMIntrinsicCall()
1922 case Intrinsic::arm_cde_vcx1q_predicated: in UpgradeARMIntrinsicCall()
1923 case Intrinsic::arm_cde_vcx1qa_predicated: in UpgradeARMIntrinsicCall()
1924 case Intrinsic::arm_cde_vcx2q_predicated: in UpgradeARMIntrinsicCall()
1925 case Intrinsic::arm_cde_vcx2qa_predicated: in UpgradeARMIntrinsicCall()
1926 case Intrinsic::arm_cde_vcx3q_predicated: in UpgradeARMIntrinsicCall()
1927 case Intrinsic::arm_cde_vcx3qa_predicated: in UpgradeARMIntrinsicCall()
1939 Intrinsic::getDeclaration( in UpgradeARMIntrinsicCall()
1940 F->getParent(), Intrinsic::arm_mve_pred_v2i, in UpgradeARMIntrinsicCall()
1944 Intrinsic::getDeclaration(F->getParent(), in UpgradeARMIntrinsicCall()
1945 Intrinsic::arm_mve_pred_i2v, {V2I1Ty}), in UpgradeARMIntrinsicCall()
1951 Function *Fn = Intrinsic::getDeclaration(F->getParent(), ID, Tys); in UpgradeARMIntrinsicCall()
2110 Function *Intr = Intrinsic::getDeclaration(F->getParent(), in UpgradeIntrinsicCall()
2111 Intrinsic::sqrt, Elt0->getType()); in UpgradeIntrinsicCall()
2117 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), in UpgradeIntrinsicCall()
2118 Intrinsic::sqrt, in UpgradeIntrinsicCall()
2125 Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512 in UpgradeIntrinsicCall()
2126 : Intrinsic::x86_avx512_sqrt_pd_512; in UpgradeIntrinsicCall()
2129 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), in UpgradeIntrinsicCall()
2132 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), in UpgradeIntrinsicCall()
2133 Intrinsic::sqrt, in UpgradeIntrinsicCall()
2247 Intrinsic::ID IID; in UpgradeIntrinsicCall()
2250 case 128: IID = Intrinsic::x86_avx512_vpshufbitqmb_128; break; in UpgradeIntrinsicCall()
2251 case 256: IID = Intrinsic::x86_avx512_vpshufbitqmb_256; break; in UpgradeIntrinsicCall()
2252 case 512: IID = Intrinsic::x86_avx512_vpshufbitqmb_512; break; in UpgradeIntrinsicCall()
2255 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in UpgradeIntrinsicCall()
2262 Intrinsic::ID IID; in UpgradeIntrinsicCall()
2264 IID = Intrinsic::x86_avx512_fpclass_ps_128; in UpgradeIntrinsicCall()
2266 IID = Intrinsic::x86_avx512_fpclass_ps_256; in UpgradeIntrinsicCall()
2268 IID = Intrinsic::x86_avx512_fpclass_ps_512; in UpgradeIntrinsicCall()
2270 IID = Intrinsic::x86_avx512_fpclass_pd_128; in UpgradeIntrinsicCall()
2272 IID = Intrinsic::x86_avx512_fpclass_pd_256; in UpgradeIntrinsicCall()
2274 IID = Intrinsic::x86_avx512_fpclass_pd_512; in UpgradeIntrinsicCall()
2278 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in UpgradeIntrinsicCall()
2286 Intrinsic::ID IID; in UpgradeIntrinsicCall()
2288 IID = Intrinsic::x86_avx512_mask_cmp_ps_128; in UpgradeIntrinsicCall()
2290 IID = Intrinsic::x86_avx512_mask_cmp_ps_256; in UpgradeIntrinsicCall()
2292 IID = Intrinsic::x86_avx512_mask_cmp_ps_512; in UpgradeIntrinsicCall()
2294 IID = Intrinsic::x86_avx512_mask_cmp_pd_128; in UpgradeIntrinsicCall()
2296 IID = Intrinsic::x86_avx512_mask_cmp_pd_256; in UpgradeIntrinsicCall()
2298 IID = Intrinsic::x86_avx512_mask_cmp_pd_512; in UpgradeIntrinsicCall()
2307 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in UpgradeIntrinsicCall()
2335 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax); in UpgradeIntrinsicCall()
2341 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax); in UpgradeIntrinsicCall()
2347 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin); in UpgradeIntrinsicCall()
2353 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin); in UpgradeIntrinsicCall()
2417 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round in UpgradeIntrinsicCall()
2418 : Intrinsic::x86_avx512_sitofp_round; in UpgradeIntrinsicCall()
2419 Function *F = Intrinsic::getDeclaration(CI->getModule(), IID, in UpgradeIntrinsicCall()
2463 Function *ELd = Intrinsic::getDeclaration(F->getParent(), in UpgradeIntrinsicCall()
2464 Intrinsic::masked_expandload, in UpgradeIntrinsicCall()
2479 Function *CSt = Intrinsic::getDeclaration(F->getParent(), in UpgradeIntrinsicCall()
2480 Intrinsic::masked_compressstore, in UpgradeIntrinsicCall()
2491 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress in UpgradeIntrinsicCall()
2492 : Intrinsic::x86_avx512_mask_expand; in UpgradeIntrinsicCall()
2493 Function *Intr = Intrinsic::getDeclaration(F->getParent(), IID, ResultTy); in UpgradeIntrinsicCall()
2557 Function *CRC32 = Intrinsic::getDeclaration(F->getParent(), in UpgradeIntrinsicCall()
2558 Intrinsic::x86_sse42_crc32_32_8); in UpgradeIntrinsicCall()
2680 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat); in UpgradeIntrinsicCall()
2685 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat); in UpgradeIntrinsicCall()
2689 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat); in UpgradeIntrinsicCall()
2693 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat); in UpgradeIntrinsicCall()
3065 Intrinsic::ID IID; in UpgradeIntrinsicCall()
3067 IID = Intrinsic::x86_avx512_add_ps_512; in UpgradeIntrinsicCall()
3069 IID = Intrinsic::x86_avx512_add_pd_512; in UpgradeIntrinsicCall()
3071 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in UpgradeIntrinsicCall()
3081 Intrinsic::ID IID; in UpgradeIntrinsicCall()
3083 IID = Intrinsic::x86_avx512_div_ps_512; in UpgradeIntrinsicCall()
3085 IID = Intrinsic::x86_avx512_div_pd_512; in UpgradeIntrinsicCall()
3087 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in UpgradeIntrinsicCall()
3097 Intrinsic::ID IID; in UpgradeIntrinsicCall()
3099 IID = Intrinsic::x86_avx512_mul_ps_512; in UpgradeIntrinsicCall()
3101 IID = Intrinsic::x86_avx512_mul_pd_512; in UpgradeIntrinsicCall()
3103 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in UpgradeIntrinsicCall()
3113 Intrinsic::ID IID; in UpgradeIntrinsicCall()
3115 IID = Intrinsic::x86_avx512_sub_ps_512; in UpgradeIntrinsicCall()
3117 IID = Intrinsic::x86_avx512_sub_pd_512; in UpgradeIntrinsicCall()
3119 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in UpgradeIntrinsicCall()
3132 static const Intrinsic::ID MinMaxTbl[2][2] = { in UpgradeIntrinsicCall()
3133 { Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512 }, in UpgradeIntrinsicCall()
3134 { Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512 } in UpgradeIntrinsicCall()
3136 Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble]; in UpgradeIntrinsicCall()
3138 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in UpgradeIntrinsicCall()
3144 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), in UpgradeIntrinsicCall()
3145 Intrinsic::ctlz, in UpgradeIntrinsicCall()
3159 Intrinsic::ID IID; in UpgradeIntrinsicCall()
3162 IID = Intrinsic::x86_avx2_psllv_q; in UpgradeIntrinsicCall()
3164 IID = Intrinsic::x86_avx2_psllv_q_256; in UpgradeIntrinsicCall()
3166 IID = Intrinsic::x86_avx2_psllv_d; in UpgradeIntrinsicCall()
3168 IID = Intrinsic::x86_avx2_psllv_d_256; in UpgradeIntrinsicCall()
3170 IID = Intrinsic::x86_avx512_psllv_w_128; in UpgradeIntrinsicCall()
3172 IID = Intrinsic::x86_avx512_psllv_w_256; in UpgradeIntrinsicCall()
3174 IID = Intrinsic::x86_avx512_psllv_w_512; in UpgradeIntrinsicCall()
3179 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d in UpgradeIntrinsicCall()
3180 : Intrinsic::x86_sse2_psll_d; in UpgradeIntrinsicCall()
3182 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q in UpgradeIntrinsicCall()
3183 : Intrinsic::x86_sse2_psll_q; in UpgradeIntrinsicCall()
3185 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w in UpgradeIntrinsicCall()
3186 : Intrinsic::x86_sse2_psll_w; in UpgradeIntrinsicCall()
3191 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d in UpgradeIntrinsicCall()
3192 : Intrinsic::x86_avx2_psll_d; in UpgradeIntrinsicCall()
3194 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q in UpgradeIntrinsicCall()
3195 : Intrinsic::x86_avx2_psll_q; in UpgradeIntrinsicCall()
3197 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w in UpgradeIntrinsicCall()
3198 : Intrinsic::x86_avx2_psll_w; in UpgradeIntrinsicCall()
3203 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512 : in UpgradeIntrinsicCall()
3204 IsVariable ? Intrinsic::x86_avx512_psllv_d_512 : in UpgradeIntrinsicCall()
3205 Intrinsic::x86_avx512_psll_d_512; in UpgradeIntrinsicCall()
3207 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512 : in UpgradeIntrinsicCall()
3208 IsVariable ? Intrinsic::x86_avx512_psllv_q_512 : in UpgradeIntrinsicCall()
3209 Intrinsic::x86_avx512_psll_q_512; in UpgradeIntrinsicCall()
3211 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512 in UpgradeIntrinsicCall()
3212 : Intrinsic::x86_avx512_psll_w_512; in UpgradeIntrinsicCall()
3227 Intrinsic::ID IID; in UpgradeIntrinsicCall()
3230 IID = Intrinsic::x86_avx2_psrlv_q; in UpgradeIntrinsicCall()
3232 IID = Intrinsic::x86_avx2_psrlv_q_256; in UpgradeIntrinsicCall()
3234 IID = Intrinsic::x86_avx2_psrlv_d; in UpgradeIntrinsicCall()
3236 IID = Intrinsic::x86_avx2_psrlv_d_256; in UpgradeIntrinsicCall()
3238 IID = Intrinsic::x86_avx512_psrlv_w_128; in UpgradeIntrinsicCall()
3240 IID = Intrinsic::x86_avx512_psrlv_w_256; in UpgradeIntrinsicCall()
3242 IID = Intrinsic::x86_avx512_psrlv_w_512; in UpgradeIntrinsicCall()
3247 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d in UpgradeIntrinsicCall()
3248 : Intrinsic::x86_sse2_psrl_d; in UpgradeIntrinsicCall()
3250 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q in UpgradeIntrinsicCall()
3251 : Intrinsic::x86_sse2_psrl_q; in UpgradeIntrinsicCall()
3253 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w in UpgradeIntrinsicCall()
3254 : Intrinsic::x86_sse2_psrl_w; in UpgradeIntrinsicCall()
3259 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d in UpgradeIntrinsicCall()
3260 : Intrinsic::x86_avx2_psrl_d; in UpgradeIntrinsicCall()
3262 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q in UpgradeIntrinsicCall()
3263 : Intrinsic::x86_avx2_psrl_q; in UpgradeIntrinsicCall()
3265 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w in UpgradeIntrinsicCall()
3266 : Intrinsic::x86_avx2_psrl_w; in UpgradeIntrinsicCall()
3271 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512 : in UpgradeIntrinsicCall()
3272 IsVariable ? Intrinsic::x86_avx512_psrlv_d_512 : in UpgradeIntrinsicCall()
3273 Intrinsic::x86_avx512_psrl_d_512; in UpgradeIntrinsicCall()
3275 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512 : in UpgradeIntrinsicCall()
3276 IsVariable ? Intrinsic::x86_avx512_psrlv_q_512 : in UpgradeIntrinsicCall()
3277 Intrinsic::x86_avx512_psrl_q_512; in UpgradeIntrinsicCall()
3279 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512 in UpgradeIntrinsicCall()
3280 : Intrinsic::x86_avx512_psrl_w_512; in UpgradeIntrinsicCall()
3295 Intrinsic::ID IID; in UpgradeIntrinsicCall()
3298 IID = Intrinsic::x86_avx2_psrav_d; in UpgradeIntrinsicCall()
3300 IID = Intrinsic::x86_avx2_psrav_d_256; in UpgradeIntrinsicCall()
3302 IID = Intrinsic::x86_avx512_psrav_w_128; in UpgradeIntrinsicCall()
3304 IID = Intrinsic::x86_avx512_psrav_w_256; in UpgradeIntrinsicCall()
3306 IID = Intrinsic::x86_avx512_psrav_w_512; in UpgradeIntrinsicCall()
3311 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d in UpgradeIntrinsicCall()
3312 : Intrinsic::x86_sse2_psra_d; in UpgradeIntrinsicCall()
3314 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128 : in UpgradeIntrinsicCall()
3315 IsVariable ? Intrinsic::x86_avx512_psrav_q_128 : in UpgradeIntrinsicCall()
3316 Intrinsic::x86_avx512_psra_q_128; in UpgradeIntrinsicCall()
3318 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w in UpgradeIntrinsicCall()
3319 : Intrinsic::x86_sse2_psra_w; in UpgradeIntrinsicCall()
3324 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d in UpgradeIntrinsicCall()
3325 : Intrinsic::x86_avx2_psra_d; in UpgradeIntrinsicCall()
3327 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256 : in UpgradeIntrinsicCall()
3328 IsVariable ? Intrinsic::x86_avx512_psrav_q_256 : in UpgradeIntrinsicCall()
3329 Intrinsic::x86_avx512_psra_q_256; in UpgradeIntrinsicCall()
3331 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w in UpgradeIntrinsicCall()
3332 : Intrinsic::x86_avx2_psra_w; in UpgradeIntrinsicCall()
3337 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512 : in UpgradeIntrinsicCall()
3338 IsVariable ? Intrinsic::x86_avx512_psrav_d_512 : in UpgradeIntrinsicCall()
3339 Intrinsic::x86_avx512_psra_d_512; in UpgradeIntrinsicCall()
3341 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512 : in UpgradeIntrinsicCall()
3342 IsVariable ? Intrinsic::x86_avx512_psrav_q_512 : in UpgradeIntrinsicCall()
3343 Intrinsic::x86_avx512_psra_q_512; in UpgradeIntrinsicCall()
3345 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512 in UpgradeIntrinsicCall()
3346 : Intrinsic::x86_avx512_psra_w_512; in UpgradeIntrinsicCall()
3395 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), in UpgradeIntrinsicCall()
3396 Intrinsic::fma, in UpgradeIntrinsicCall()
3411 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), in UpgradeIntrinsicCall()
3412 Intrinsic::fma, in UpgradeIntrinsicCall()
3449 Intrinsic::ID IID; in UpgradeIntrinsicCall()
3451 IID = Intrinsic::x86_avx512_vfmadd_f64; in UpgradeIntrinsicCall()
3453 IID = Intrinsic::x86_avx512_vfmadd_f32; in UpgradeIntrinsicCall()
3454 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), IID); in UpgradeIntrinsicCall()
3457 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), in UpgradeIntrinsicCall()
3458 Intrinsic::fma, in UpgradeIntrinsicCall()
3504 Intrinsic::ID IID; in UpgradeIntrinsicCall()
3507 IID = Intrinsic::x86_avx512_vfmadd_ps_512; in UpgradeIntrinsicCall()
3509 IID = Intrinsic::x86_avx512_vfmadd_pd_512; in UpgradeIntrinsicCall()
3511 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in UpgradeIntrinsicCall()
3514 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), in UpgradeIntrinsicCall()
3515 Intrinsic::fma, in UpgradeIntrinsicCall()
3528 Intrinsic::ID IID; in UpgradeIntrinsicCall()
3530 IID = Intrinsic::x86_fma_vfmaddsub_ps; in UpgradeIntrinsicCall()
3532 IID = Intrinsic::x86_fma_vfmaddsub_ps_256; in UpgradeIntrinsicCall()
3534 IID = Intrinsic::x86_fma_vfmaddsub_pd; in UpgradeIntrinsicCall()
3536 IID = Intrinsic::x86_fma_vfmaddsub_pd_256; in UpgradeIntrinsicCall()
3543 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in UpgradeIntrinsicCall()
3555 Intrinsic::ID IID; in UpgradeIntrinsicCall()
3558 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512; in UpgradeIntrinsicCall()
3560 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512; in UpgradeIntrinsicCall()
3567 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in UpgradeIntrinsicCall()
3575 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma, in UpgradeIntrinsicCall()
3601 Intrinsic::ID IID; in UpgradeIntrinsicCall()
3603 IID = Intrinsic::x86_avx512_pternlog_d_128; in UpgradeIntrinsicCall()
3605 IID = Intrinsic::x86_avx512_pternlog_d_256; in UpgradeIntrinsicCall()
3607 IID = Intrinsic::x86_avx512_pternlog_d_512; in UpgradeIntrinsicCall()
3609 IID = Intrinsic::x86_avx512_pternlog_q_128; in UpgradeIntrinsicCall()
3611 IID = Intrinsic::x86_avx512_pternlog_q_256; in UpgradeIntrinsicCall()
3613 IID = Intrinsic::x86_avx512_pternlog_q_512; in UpgradeIntrinsicCall()
3619 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in UpgradeIntrinsicCall()
3629 Intrinsic::ID IID; in UpgradeIntrinsicCall()
3631 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128; in UpgradeIntrinsicCall()
3633 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256; in UpgradeIntrinsicCall()
3635 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512; in UpgradeIntrinsicCall()
3637 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128; in UpgradeIntrinsicCall()
3639 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256; in UpgradeIntrinsicCall()
3641 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512; in UpgradeIntrinsicCall()
3647 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in UpgradeIntrinsicCall()
3665 Intrinsic::ID IID; in UpgradeIntrinsicCall()
3667 IID = Intrinsic::x86_avx512_vpdpbusd_128; in UpgradeIntrinsicCall()
3669 IID = Intrinsic::x86_avx512_vpdpbusd_256; in UpgradeIntrinsicCall()
3671 IID = Intrinsic::x86_avx512_vpdpbusd_512; in UpgradeIntrinsicCall()
3673 IID = Intrinsic::x86_avx512_vpdpbusds_128; in UpgradeIntrinsicCall()
3675 IID = Intrinsic::x86_avx512_vpdpbusds_256; in UpgradeIntrinsicCall()
3677 IID = Intrinsic::x86_avx512_vpdpbusds_512; in UpgradeIntrinsicCall()
3683 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in UpgradeIntrinsicCall()
3695 Intrinsic::ID IID; in UpgradeIntrinsicCall()
3697 IID = Intrinsic::x86_avx512_vpdpwssd_128; in UpgradeIntrinsicCall()
3699 IID = Intrinsic::x86_avx512_vpdpwssd_256; in UpgradeIntrinsicCall()
3701 IID = Intrinsic::x86_avx512_vpdpwssd_512; in UpgradeIntrinsicCall()
3703 IID = Intrinsic::x86_avx512_vpdpwssds_128; in UpgradeIntrinsicCall()
3705 IID = Intrinsic::x86_avx512_vpdpwssds_256; in UpgradeIntrinsicCall()
3707 IID = Intrinsic::x86_avx512_vpdpwssds_512; in UpgradeIntrinsicCall()
3713 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in UpgradeIntrinsicCall()
3721 Intrinsic::ID IID; in UpgradeIntrinsicCall()
3723 IID = Intrinsic::x86_addcarry_32; in UpgradeIntrinsicCall()
3725 IID = Intrinsic::x86_addcarry_64; in UpgradeIntrinsicCall()
3727 IID = Intrinsic::x86_subborrow_32; in UpgradeIntrinsicCall()
3729 IID = Intrinsic::x86_subborrow_64; in UpgradeIntrinsicCall()
3737 Intrinsic::getDeclaration(CI->getModule(), IID), in UpgradeIntrinsicCall()
3786 Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz, in UpgradeIntrinsicCall()
3795 Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop, in UpgradeIntrinsicCall()
3800 Rep = Builder.CreateCall(Intrinsic::getDeclaration( in UpgradeIntrinsicCall()
3801 F->getParent(), Intrinsic::convert_from_fp16, in UpgradeIntrinsicCall()
3850 case Intrinsic::arm_neon_vst1: in UpgradeIntrinsicCall()
3851 case Intrinsic::arm_neon_vst2: in UpgradeIntrinsicCall()
3852 case Intrinsic::arm_neon_vst3: in UpgradeIntrinsicCall()
3853 case Intrinsic::arm_neon_vst4: in UpgradeIntrinsicCall()
3854 case Intrinsic::arm_neon_vst2lane: in UpgradeIntrinsicCall()
3855 case Intrinsic::arm_neon_vst3lane: in UpgradeIntrinsicCall()
3856 case Intrinsic::arm_neon_vst4lane: { in UpgradeIntrinsicCall()
3862 case Intrinsic::arm_neon_bfdot: in UpgradeIntrinsicCall()
3863 case Intrinsic::arm_neon_bfmmla: in UpgradeIntrinsicCall()
3864 case Intrinsic::arm_neon_bfmlalb: in UpgradeIntrinsicCall()
3865 case Intrinsic::arm_neon_bfmlalt: in UpgradeIntrinsicCall()
3866 case Intrinsic::aarch64_neon_bfdot: in UpgradeIntrinsicCall()
3867 case Intrinsic::aarch64_neon_bfmmla: in UpgradeIntrinsicCall()
3868 case Intrinsic::aarch64_neon_bfmlalb: in UpgradeIntrinsicCall()
3869 case Intrinsic::aarch64_neon_bfmlalt: { in UpgradeIntrinsicCall()
3886 case Intrinsic::bitreverse: in UpgradeIntrinsicCall()
3890 case Intrinsic::ctlz: in UpgradeIntrinsicCall()
3891 case Intrinsic::cttz: in UpgradeIntrinsicCall()
3898 case Intrinsic::objectsize: { in UpgradeIntrinsicCall()
3908 case Intrinsic::ctpop: in UpgradeIntrinsicCall()
3912 case Intrinsic::convert_from_fp16: in UpgradeIntrinsicCall()
3916 case Intrinsic::dbg_value: in UpgradeIntrinsicCall()
3930 case Intrinsic::ptr_annotation: in UpgradeIntrinsicCall()
3947 case Intrinsic::var_annotation: in UpgradeIntrinsicCall()
3959 case Intrinsic::x86_xop_vfrcz_ss: in UpgradeIntrinsicCall()
3960 case Intrinsic::x86_xop_vfrcz_sd: in UpgradeIntrinsicCall()
3964 case Intrinsic::x86_xop_vpermil2pd: in UpgradeIntrinsicCall()
3965 case Intrinsic::x86_xop_vpermil2ps: in UpgradeIntrinsicCall()
3966 case Intrinsic::x86_xop_vpermil2pd_256: in UpgradeIntrinsicCall()
3967 case Intrinsic::x86_xop_vpermil2ps_256: { in UpgradeIntrinsicCall()
3976 case Intrinsic::x86_sse41_ptestc: in UpgradeIntrinsicCall()
3977 case Intrinsic::x86_sse41_ptestz: in UpgradeIntrinsicCall()
3978 case Intrinsic::x86_sse41_ptestnzc: { in UpgradeIntrinsicCall()
3999 case Intrinsic::x86_rdtscp: { in UpgradeIntrinsicCall()
4021 case Intrinsic::x86_sse41_insertps: in UpgradeIntrinsicCall()
4022 case Intrinsic::x86_sse41_dppd: in UpgradeIntrinsicCall()
4023 case Intrinsic::x86_sse41_dpps: in UpgradeIntrinsicCall()
4024 case Intrinsic::x86_sse41_mpsadbw: in UpgradeIntrinsicCall()
4025 case Intrinsic::x86_avx_dp_ps_256: in UpgradeIntrinsicCall()
4026 case Intrinsic::x86_avx2_mpsadbw: { in UpgradeIntrinsicCall()
4037 case Intrinsic::x86_avx512_mask_cmp_pd_128: in UpgradeIntrinsicCall()
4038 case Intrinsic::x86_avx512_mask_cmp_pd_256: in UpgradeIntrinsicCall()
4039 case Intrinsic::x86_avx512_mask_cmp_pd_512: in UpgradeIntrinsicCall()
4040 case Intrinsic::x86_avx512_mask_cmp_ps_128: in UpgradeIntrinsicCall()
4041 case Intrinsic::x86_avx512_mask_cmp_ps_256: in UpgradeIntrinsicCall()
4042 case Intrinsic::x86_avx512_mask_cmp_ps_512: { in UpgradeIntrinsicCall()
4057 case Intrinsic::thread_pointer: { in UpgradeIntrinsicCall()
4062 case Intrinsic::invariant_start: in UpgradeIntrinsicCall()
4063 case Intrinsic::invariant_end: { in UpgradeIntrinsicCall()
4068 case Intrinsic::masked_load: in UpgradeIntrinsicCall()
4069 case Intrinsic::masked_store: in UpgradeIntrinsicCall()
4070 case Intrinsic::masked_gather: in UpgradeIntrinsicCall()
4071 case Intrinsic::masked_scatter: { in UpgradeIntrinsicCall()
4078 case Intrinsic::memcpy: in UpgradeIntrinsicCall()
4079 case Intrinsic::memmove: in UpgradeIntrinsicCall()
4080 case Intrinsic::memset: { in UpgradeIntrinsicCall()
4258 llvm::Intrinsic::ID IntrinsicFunc) { in UpgradeARCRuntime()
4264 Function *NewFn = llvm::Intrinsic::getDeclaration(&M, IntrinsicFunc); in UpgradeARCRuntime()
4324 UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use); in UpgradeARCRuntime()
4332 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = { in UpgradeARCRuntime()
4333 {"objc_autorelease", llvm::Intrinsic::objc_autorelease}, in UpgradeARCRuntime()
4334 {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop}, in UpgradeARCRuntime()
4335 {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush}, in UpgradeARCRuntime()
4337 llvm::Intrinsic::objc_autoreleaseReturnValue}, in UpgradeARCRuntime()
4338 {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak}, in UpgradeARCRuntime()
4339 {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak}, in UpgradeARCRuntime()
4340 {"objc_initWeak", llvm::Intrinsic::objc_initWeak}, in UpgradeARCRuntime()
4341 {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak}, in UpgradeARCRuntime()
4342 {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained}, in UpgradeARCRuntime()
4343 {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak}, in UpgradeARCRuntime()
4344 {"objc_release", llvm::Intrinsic::objc_release}, in UpgradeARCRuntime()
4345 {"objc_retain", llvm::Intrinsic::objc_retain}, in UpgradeARCRuntime()
4346 {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease}, in UpgradeARCRuntime()
4348 llvm::Intrinsic::objc_retainAutoreleaseReturnValue}, in UpgradeARCRuntime()
4350 llvm::Intrinsic::objc_retainAutoreleasedReturnValue}, in UpgradeARCRuntime()
4351 {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock}, in UpgradeARCRuntime()
4352 {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong}, in UpgradeARCRuntime()
4353 {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak}, in UpgradeARCRuntime()
4355 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue}, in UpgradeARCRuntime()
4356 {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject}, in UpgradeARCRuntime()
4357 {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject}, in UpgradeARCRuntime()
4358 {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer}, in UpgradeARCRuntime()
4359 {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease}, in UpgradeARCRuntime()
4360 {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter}, in UpgradeARCRuntime()
4361 {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit}, in UpgradeARCRuntime()
4363 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart}, in UpgradeARCRuntime()
4365 llvm::Intrinsic::objc_arc_annotation_topdown_bbend}, in UpgradeARCRuntime()
4367 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart}, in UpgradeARCRuntime()
4369 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}}; in UpgradeARCRuntime()