Lines Matching refs:IID
56 static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID, in upgradePTESTIntrinsic() argument
66 NewFn = Intrinsic::getDeclaration(F->getParent(), IID); in upgradePTESTIntrinsic()
72 static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, in upgradeX86IntrinsicsWith8BitMask() argument
82 NewFn = Intrinsic::getDeclaration(F->getParent(), IID); in upgradeX86IntrinsicsWith8BitMask()
88 static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID, in upgradeX86MaskedFPCompare() argument
95 NewFn = Intrinsic::getDeclaration(F->getParent(), IID); in upgradeX86MaskedFPCompare()
99 static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID, in upgradeX86BF16Intrinsic() argument
105 NewFn = Intrinsic::getDeclaration(F->getParent(), IID); in upgradeX86BF16Intrinsic()
109 static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID, in upgradeX86BF16DPIntrinsic() argument
115 NewFn = Intrinsic::getDeclaration(F->getParent(), IID); in upgradeX86BF16DPIntrinsic()
1178 Intrinsic::ID IID = in upgradeIntrinsicFunction1() local
1184 if (IID != Intrinsic::not_intrinsic) { in upgradeIntrinsicFunction1()
1185 NewFn = Intrinsic::getDeclaration(F->getParent(), IID, in upgradeIntrinsicFunction1()
1193 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name); in upgradeIntrinsicFunction1() local
1194 if (IID != Intrinsic::not_intrinsic) { in upgradeIntrinsicFunction1()
1612 Intrinsic::ID IID; in upgradeX86VPERMT2Intrinsics() local
1614 IID = Intrinsic::x86_avx512_vpermi2var_ps_128; in upgradeX86VPERMT2Intrinsics()
1616 IID = Intrinsic::x86_avx512_vpermi2var_d_128; in upgradeX86VPERMT2Intrinsics()
1618 IID = Intrinsic::x86_avx512_vpermi2var_pd_128; in upgradeX86VPERMT2Intrinsics()
1620 IID = Intrinsic::x86_avx512_vpermi2var_q_128; in upgradeX86VPERMT2Intrinsics()
1622 IID = Intrinsic::x86_avx512_vpermi2var_ps_256; in upgradeX86VPERMT2Intrinsics()
1624 IID = Intrinsic::x86_avx512_vpermi2var_d_256; in upgradeX86VPERMT2Intrinsics()
1626 IID = Intrinsic::x86_avx512_vpermi2var_pd_256; in upgradeX86VPERMT2Intrinsics()
1628 IID = Intrinsic::x86_avx512_vpermi2var_q_256; in upgradeX86VPERMT2Intrinsics()
1630 IID = Intrinsic::x86_avx512_vpermi2var_ps_512; in upgradeX86VPERMT2Intrinsics()
1632 IID = Intrinsic::x86_avx512_vpermi2var_d_512; in upgradeX86VPERMT2Intrinsics()
1634 IID = Intrinsic::x86_avx512_vpermi2var_pd_512; in upgradeX86VPERMT2Intrinsics()
1636 IID = Intrinsic::x86_avx512_vpermi2var_q_512; in upgradeX86VPERMT2Intrinsics()
1638 IID = Intrinsic::x86_avx512_vpermi2var_hi_128; in upgradeX86VPERMT2Intrinsics()
1640 IID = Intrinsic::x86_avx512_vpermi2var_hi_256; in upgradeX86VPERMT2Intrinsics()
1642 IID = Intrinsic::x86_avx512_vpermi2var_hi_512; in upgradeX86VPERMT2Intrinsics()
1644 IID = Intrinsic::x86_avx512_vpermi2var_qi_128; in upgradeX86VPERMT2Intrinsics()
1646 IID = Intrinsic::x86_avx512_vpermi2var_qi_256; in upgradeX86VPERMT2Intrinsics()
1648 IID = Intrinsic::x86_avx512_vpermi2var_qi_512; in upgradeX86VPERMT2Intrinsics()
1659 Value *V = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID), in upgradeX86VPERMT2Intrinsics()
1668 Intrinsic::ID IID) { in upgradeX86BinaryIntrinsics() argument
1672 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); in upgradeX86BinaryIntrinsics()
1698 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl; in upgradeX86Rotate() local
1699 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); in upgradeX86Rotate()
1768 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl; in upgradeX86ConcatShift() local
1769 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); in upgradeX86ConcatShift()
1922 Intrinsic::ID IID) { in upgradeX86MaskedShift() argument
1923 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID); in upgradeX86MaskedShift()
1958 Intrinsic::ID IID; in upgradeAVX512MaskToSelect() local
1961 IID = Intrinsic::x86_sse_max_ps; in upgradeAVX512MaskToSelect()
1963 IID = Intrinsic::x86_sse2_max_pd; in upgradeAVX512MaskToSelect()
1965 IID = Intrinsic::x86_avx_max_ps_256; in upgradeAVX512MaskToSelect()
1967 IID = Intrinsic::x86_avx_max_pd_256; in upgradeAVX512MaskToSelect()
1972 IID = Intrinsic::x86_sse_min_ps; in upgradeAVX512MaskToSelect()
1974 IID = Intrinsic::x86_sse2_min_pd; in upgradeAVX512MaskToSelect()
1976 IID = Intrinsic::x86_avx_min_ps_256; in upgradeAVX512MaskToSelect()
1978 IID = Intrinsic::x86_avx_min_pd_256; in upgradeAVX512MaskToSelect()
1983 IID = Intrinsic::x86_ssse3_pshuf_b_128; in upgradeAVX512MaskToSelect()
1985 IID = Intrinsic::x86_avx2_pshuf_b; in upgradeAVX512MaskToSelect()
1987 IID = Intrinsic::x86_avx512_pshuf_b_512; in upgradeAVX512MaskToSelect()
1992 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128; in upgradeAVX512MaskToSelect()
1994 IID = Intrinsic::x86_avx2_pmul_hr_sw; in upgradeAVX512MaskToSelect()
1996 IID = Intrinsic::x86_avx512_pmul_hr_sw_512; in upgradeAVX512MaskToSelect()
2001 IID = Intrinsic::x86_sse2_pmulh_w; in upgradeAVX512MaskToSelect()
2003 IID = Intrinsic::x86_avx2_pmulh_w; in upgradeAVX512MaskToSelect()
2005 IID = Intrinsic::x86_avx512_pmulh_w_512; in upgradeAVX512MaskToSelect()
2010 IID = Intrinsic::x86_sse2_pmulhu_w; in upgradeAVX512MaskToSelect()
2012 IID = Intrinsic::x86_avx2_pmulhu_w; in upgradeAVX512MaskToSelect()
2014 IID = Intrinsic::x86_avx512_pmulhu_w_512; in upgradeAVX512MaskToSelect()
2019 IID = Intrinsic::x86_sse2_pmadd_wd; in upgradeAVX512MaskToSelect()
2021 IID = Intrinsic::x86_avx2_pmadd_wd; in upgradeAVX512MaskToSelect()
2023 IID = Intrinsic::x86_avx512_pmaddw_d_512; in upgradeAVX512MaskToSelect()
2028 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128; in upgradeAVX512MaskToSelect()
2030 IID = Intrinsic::x86_avx2_pmadd_ub_sw; in upgradeAVX512MaskToSelect()
2032 IID = Intrinsic::x86_avx512_pmaddubs_w_512; in upgradeAVX512MaskToSelect()
2037 IID = Intrinsic::x86_sse2_packsswb_128; in upgradeAVX512MaskToSelect()
2039 IID = Intrinsic::x86_avx2_packsswb; in upgradeAVX512MaskToSelect()
2041 IID = Intrinsic::x86_avx512_packsswb_512; in upgradeAVX512MaskToSelect()
2046 IID = Intrinsic::x86_sse2_packssdw_128; in upgradeAVX512MaskToSelect()
2048 IID = Intrinsic::x86_avx2_packssdw; in upgradeAVX512MaskToSelect()
2050 IID = Intrinsic::x86_avx512_packssdw_512; in upgradeAVX512MaskToSelect()
2055 IID = Intrinsic::x86_sse2_packuswb_128; in upgradeAVX512MaskToSelect()
2057 IID = Intrinsic::x86_avx2_packuswb; in upgradeAVX512MaskToSelect()
2059 IID = Intrinsic::x86_avx512_packuswb_512; in upgradeAVX512MaskToSelect()
2064 IID = Intrinsic::x86_sse41_packusdw; in upgradeAVX512MaskToSelect()
2066 IID = Intrinsic::x86_avx2_packusdw; in upgradeAVX512MaskToSelect()
2068 IID = Intrinsic::x86_avx512_packusdw_512; in upgradeAVX512MaskToSelect()
2073 IID = Intrinsic::x86_avx_vpermilvar_ps; in upgradeAVX512MaskToSelect()
2075 IID = Intrinsic::x86_avx_vpermilvar_pd; in upgradeAVX512MaskToSelect()
2077 IID = Intrinsic::x86_avx_vpermilvar_ps_256; in upgradeAVX512MaskToSelect()
2079 IID = Intrinsic::x86_avx_vpermilvar_pd_256; in upgradeAVX512MaskToSelect()
2081 IID = Intrinsic::x86_avx512_vpermilvar_ps_512; in upgradeAVX512MaskToSelect()
2083 IID = Intrinsic::x86_avx512_vpermilvar_pd_512; in upgradeAVX512MaskToSelect()
2087 IID = Intrinsic::x86_avx_cvt_pd2dq_256; in upgradeAVX512MaskToSelect()
2089 IID = Intrinsic::x86_avx_cvt_pd2_ps_256; in upgradeAVX512MaskToSelect()
2091 IID = Intrinsic::x86_avx_cvtt_pd2dq_256; in upgradeAVX512MaskToSelect()
2093 IID = Intrinsic::x86_sse2_cvttps2dq; in upgradeAVX512MaskToSelect()
2095 IID = Intrinsic::x86_avx_cvtt_ps2dq_256; in upgradeAVX512MaskToSelect()
2099 IID = Intrinsic::x86_avx2_permps; in upgradeAVX512MaskToSelect()
2101 IID = Intrinsic::x86_avx2_permd; in upgradeAVX512MaskToSelect()
2103 IID = Intrinsic::x86_avx512_permvar_df_256; in upgradeAVX512MaskToSelect()
2105 IID = Intrinsic::x86_avx512_permvar_di_256; in upgradeAVX512MaskToSelect()
2107 IID = Intrinsic::x86_avx512_permvar_sf_512; in upgradeAVX512MaskToSelect()
2109 IID = Intrinsic::x86_avx512_permvar_si_512; in upgradeAVX512MaskToSelect()
2111 IID = Intrinsic::x86_avx512_permvar_df_512; in upgradeAVX512MaskToSelect()
2113 IID = Intrinsic::x86_avx512_permvar_di_512; in upgradeAVX512MaskToSelect()
2115 IID = Intrinsic::x86_avx512_permvar_hi_128; in upgradeAVX512MaskToSelect()
2117 IID = Intrinsic::x86_avx512_permvar_hi_256; in upgradeAVX512MaskToSelect()
2119 IID = Intrinsic::x86_avx512_permvar_hi_512; in upgradeAVX512MaskToSelect()
2121 IID = Intrinsic::x86_avx512_permvar_qi_128; in upgradeAVX512MaskToSelect()
2123 IID = Intrinsic::x86_avx512_permvar_qi_256; in upgradeAVX512MaskToSelect()
2125 IID = Intrinsic::x86_avx512_permvar_qi_512; in upgradeAVX512MaskToSelect()
2130 IID = Intrinsic::x86_avx512_dbpsadbw_128; in upgradeAVX512MaskToSelect()
2132 IID = Intrinsic::x86_avx512_dbpsadbw_256; in upgradeAVX512MaskToSelect()
2134 IID = Intrinsic::x86_avx512_dbpsadbw_512; in upgradeAVX512MaskToSelect()
2139 IID = Intrinsic::x86_avx512_pmultishift_qb_128; in upgradeAVX512MaskToSelect()
2141 IID = Intrinsic::x86_avx512_pmultishift_qb_256; in upgradeAVX512MaskToSelect()
2143 IID = Intrinsic::x86_avx512_pmultishift_qb_512; in upgradeAVX512MaskToSelect()
2148 IID = Intrinsic::x86_avx512_conflict_d_128; in upgradeAVX512MaskToSelect()
2150 IID = Intrinsic::x86_avx512_conflict_d_256; in upgradeAVX512MaskToSelect()
2152 IID = Intrinsic::x86_avx512_conflict_d_512; in upgradeAVX512MaskToSelect()
2154 IID = Intrinsic::x86_avx512_conflict_q_128; in upgradeAVX512MaskToSelect()
2156 IID = Intrinsic::x86_avx512_conflict_q_256; in upgradeAVX512MaskToSelect()
2158 IID = Intrinsic::x86_avx512_conflict_q_512; in upgradeAVX512MaskToSelect()
2163 IID = Intrinsic::x86_sse2_pavg_b; in upgradeAVX512MaskToSelect()
2165 IID = Intrinsic::x86_avx2_pavg_b; in upgradeAVX512MaskToSelect()
2167 IID = Intrinsic::x86_avx512_pavg_b_512; in upgradeAVX512MaskToSelect()
2169 IID = Intrinsic::x86_sse2_pavg_w; in upgradeAVX512MaskToSelect()
2171 IID = Intrinsic::x86_avx2_pavg_w; in upgradeAVX512MaskToSelect()
2173 IID = Intrinsic::x86_avx512_pavg_w_512; in upgradeAVX512MaskToSelect()
2182 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID), in upgradeAVX512MaskToSelect()
2496 Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512 in UpgradeIntrinsicCall() local
2501 IID), Args); in UpgradeIntrinsicCall()
2617 Intrinsic::ID IID; in UpgradeIntrinsicCall() local
2620 case 128: IID = Intrinsic::x86_avx512_vpshufbitqmb_128; break; in UpgradeIntrinsicCall()
2621 case 256: IID = Intrinsic::x86_avx512_vpshufbitqmb_256; break; in UpgradeIntrinsicCall()
2622 case 512: IID = Intrinsic::x86_avx512_vpshufbitqmb_512; break; in UpgradeIntrinsicCall()
2625 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in UpgradeIntrinsicCall()
2632 Intrinsic::ID IID; in UpgradeIntrinsicCall() local
2634 IID = Intrinsic::x86_avx512_fpclass_ps_128; in UpgradeIntrinsicCall()
2636 IID = Intrinsic::x86_avx512_fpclass_ps_256; in UpgradeIntrinsicCall()
2638 IID = Intrinsic::x86_avx512_fpclass_ps_512; in UpgradeIntrinsicCall()
2640 IID = Intrinsic::x86_avx512_fpclass_pd_128; in UpgradeIntrinsicCall()
2642 IID = Intrinsic::x86_avx512_fpclass_pd_256; in UpgradeIntrinsicCall()
2644 IID = Intrinsic::x86_avx512_fpclass_pd_512; in UpgradeIntrinsicCall()
2648 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in UpgradeIntrinsicCall()
2656 Intrinsic::ID IID; in UpgradeIntrinsicCall() local
2658 IID = Intrinsic::x86_avx512_mask_cmp_ps_128; in UpgradeIntrinsicCall()
2660 IID = Intrinsic::x86_avx512_mask_cmp_ps_256; in UpgradeIntrinsicCall()
2662 IID = Intrinsic::x86_avx512_mask_cmp_ps_512; in UpgradeIntrinsicCall()
2664 IID = Intrinsic::x86_avx512_mask_cmp_pd_128; in UpgradeIntrinsicCall()
2666 IID = Intrinsic::x86_avx512_mask_cmp_pd_256; in UpgradeIntrinsicCall()
2668 IID = Intrinsic::x86_avx512_mask_cmp_pd_512; in UpgradeIntrinsicCall()
2677 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in UpgradeIntrinsicCall()
2787 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round in UpgradeIntrinsicCall() local
2789 Function *F = Intrinsic::getDeclaration(CI->getModule(), IID, in UpgradeIntrinsicCall()
2861 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress in UpgradeIntrinsicCall() local
2863 Function *Intr = Intrinsic::getDeclaration(F->getParent(), IID, ResultTy); in UpgradeIntrinsicCall()
3429 Intrinsic::ID IID; in UpgradeIntrinsicCall() local
3431 IID = Intrinsic::x86_avx512_add_ps_512; in UpgradeIntrinsicCall()
3433 IID = Intrinsic::x86_avx512_add_pd_512; in UpgradeIntrinsicCall()
3435 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in UpgradeIntrinsicCall()
3445 Intrinsic::ID IID; in UpgradeIntrinsicCall() local
3447 IID = Intrinsic::x86_avx512_div_ps_512; in UpgradeIntrinsicCall()
3449 IID = Intrinsic::x86_avx512_div_pd_512; in UpgradeIntrinsicCall()
3451 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in UpgradeIntrinsicCall()
3461 Intrinsic::ID IID; in UpgradeIntrinsicCall() local
3463 IID = Intrinsic::x86_avx512_mul_ps_512; in UpgradeIntrinsicCall()
3465 IID = Intrinsic::x86_avx512_mul_pd_512; in UpgradeIntrinsicCall()
3467 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in UpgradeIntrinsicCall()
3477 Intrinsic::ID IID; in UpgradeIntrinsicCall() local
3479 IID = Intrinsic::x86_avx512_sub_ps_512; in UpgradeIntrinsicCall()
3481 IID = Intrinsic::x86_avx512_sub_pd_512; in UpgradeIntrinsicCall()
3483 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in UpgradeIntrinsicCall()
3500 Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble]; in UpgradeIntrinsicCall() local
3502 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in UpgradeIntrinsicCall()
3523 Intrinsic::ID IID; in UpgradeIntrinsicCall() local
3526 IID = Intrinsic::x86_avx2_psllv_q; in UpgradeIntrinsicCall()
3528 IID = Intrinsic::x86_avx2_psllv_q_256; in UpgradeIntrinsicCall()
3530 IID = Intrinsic::x86_avx2_psllv_d; in UpgradeIntrinsicCall()
3532 IID = Intrinsic::x86_avx2_psllv_d_256; in UpgradeIntrinsicCall()
3534 IID = Intrinsic::x86_avx512_psllv_w_128; in UpgradeIntrinsicCall()
3536 IID = Intrinsic::x86_avx512_psllv_w_256; in UpgradeIntrinsicCall()
3538 IID = Intrinsic::x86_avx512_psllv_w_512; in UpgradeIntrinsicCall()
3543 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d in UpgradeIntrinsicCall()
3546 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q in UpgradeIntrinsicCall()
3549 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w in UpgradeIntrinsicCall()
3555 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d in UpgradeIntrinsicCall()
3558 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q in UpgradeIntrinsicCall()
3561 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w in UpgradeIntrinsicCall()
3567 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512 : in UpgradeIntrinsicCall()
3571 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512 : in UpgradeIntrinsicCall()
3575 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512 in UpgradeIntrinsicCall()
3581 Rep = upgradeX86MaskedShift(Builder, *CI, IID); in UpgradeIntrinsicCall()
3591 Intrinsic::ID IID; in UpgradeIntrinsicCall() local
3594 IID = Intrinsic::x86_avx2_psrlv_q; in UpgradeIntrinsicCall()
3596 IID = Intrinsic::x86_avx2_psrlv_q_256; in UpgradeIntrinsicCall()
3598 IID = Intrinsic::x86_avx2_psrlv_d; in UpgradeIntrinsicCall()
3600 IID = Intrinsic::x86_avx2_psrlv_d_256; in UpgradeIntrinsicCall()
3602 IID = Intrinsic::x86_avx512_psrlv_w_128; in UpgradeIntrinsicCall()
3604 IID = Intrinsic::x86_avx512_psrlv_w_256; in UpgradeIntrinsicCall()
3606 IID = Intrinsic::x86_avx512_psrlv_w_512; in UpgradeIntrinsicCall()
3611 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d in UpgradeIntrinsicCall()
3614 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q in UpgradeIntrinsicCall()
3617 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w in UpgradeIntrinsicCall()
3623 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d in UpgradeIntrinsicCall()
3626 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q in UpgradeIntrinsicCall()
3629 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w in UpgradeIntrinsicCall()
3635 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512 : in UpgradeIntrinsicCall()
3639 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512 : in UpgradeIntrinsicCall()
3643 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512 in UpgradeIntrinsicCall()
3649 Rep = upgradeX86MaskedShift(Builder, *CI, IID); in UpgradeIntrinsicCall()
3659 Intrinsic::ID IID; in UpgradeIntrinsicCall() local
3662 IID = Intrinsic::x86_avx2_psrav_d; in UpgradeIntrinsicCall()
3664 IID = Intrinsic::x86_avx2_psrav_d_256; in UpgradeIntrinsicCall()
3666 IID = Intrinsic::x86_avx512_psrav_w_128; in UpgradeIntrinsicCall()
3668 IID = Intrinsic::x86_avx512_psrav_w_256; in UpgradeIntrinsicCall()
3670 IID = Intrinsic::x86_avx512_psrav_w_512; in UpgradeIntrinsicCall()
3675 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d in UpgradeIntrinsicCall()
3678 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128 : in UpgradeIntrinsicCall()
3682 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w in UpgradeIntrinsicCall()
3688 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d in UpgradeIntrinsicCall()
3691 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256 : in UpgradeIntrinsicCall()
3695 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w in UpgradeIntrinsicCall()
3701 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512 : in UpgradeIntrinsicCall()
3705 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512 : in UpgradeIntrinsicCall()
3709 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512 in UpgradeIntrinsicCall()
3715 Rep = upgradeX86MaskedShift(Builder, *CI, IID); in UpgradeIntrinsicCall()
3812 Intrinsic::ID IID; in UpgradeIntrinsicCall() local
3814 IID = Intrinsic::x86_avx512_vfmadd_f64; in UpgradeIntrinsicCall()
3816 IID = Intrinsic::x86_avx512_vfmadd_f32; in UpgradeIntrinsicCall()
3817 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), IID); in UpgradeIntrinsicCall()
3866 Intrinsic::ID IID; in UpgradeIntrinsicCall() local
3869 IID = Intrinsic::x86_avx512_vfmadd_ps_512; in UpgradeIntrinsicCall()
3871 IID = Intrinsic::x86_avx512_vfmadd_pd_512; in UpgradeIntrinsicCall()
3873 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in UpgradeIntrinsicCall()
3890 Intrinsic::ID IID; in UpgradeIntrinsicCall() local
3892 IID = Intrinsic::x86_fma_vfmaddsub_ps; in UpgradeIntrinsicCall()
3894 IID = Intrinsic::x86_fma_vfmaddsub_ps_256; in UpgradeIntrinsicCall()
3896 IID = Intrinsic::x86_fma_vfmaddsub_pd; in UpgradeIntrinsicCall()
3898 IID = Intrinsic::x86_fma_vfmaddsub_pd_256; in UpgradeIntrinsicCall()
3905 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in UpgradeIntrinsicCall()
3917 Intrinsic::ID IID; in UpgradeIntrinsicCall() local
3920 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512; in UpgradeIntrinsicCall()
3922 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512; in UpgradeIntrinsicCall()
3929 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), in UpgradeIntrinsicCall()
3963 Intrinsic::ID IID; in UpgradeIntrinsicCall() local
3965 IID = Intrinsic::x86_avx512_pternlog_d_128; in UpgradeIntrinsicCall()
3967 IID = Intrinsic::x86_avx512_pternlog_d_256; in UpgradeIntrinsicCall()
3969 IID = Intrinsic::x86_avx512_pternlog_d_512; in UpgradeIntrinsicCall()
3971 IID = Intrinsic::x86_avx512_pternlog_q_128; in UpgradeIntrinsicCall()
3973 IID = Intrinsic::x86_avx512_pternlog_q_256; in UpgradeIntrinsicCall()
3975 IID = Intrinsic::x86_avx512_pternlog_q_512; in UpgradeIntrinsicCall()
3981 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in UpgradeIntrinsicCall()
3991 Intrinsic::ID IID; in UpgradeIntrinsicCall() local
3993 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128; in UpgradeIntrinsicCall()
3995 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256; in UpgradeIntrinsicCall()
3997 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512; in UpgradeIntrinsicCall()
3999 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128; in UpgradeIntrinsicCall()
4001 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256; in UpgradeIntrinsicCall()
4003 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512; in UpgradeIntrinsicCall()
4009 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in UpgradeIntrinsicCall()
4027 Intrinsic::ID IID; in UpgradeIntrinsicCall() local
4029 IID = Intrinsic::x86_avx512_vpdpbusd_128; in UpgradeIntrinsicCall()
4031 IID = Intrinsic::x86_avx512_vpdpbusd_256; in UpgradeIntrinsicCall()
4033 IID = Intrinsic::x86_avx512_vpdpbusd_512; in UpgradeIntrinsicCall()
4035 IID = Intrinsic::x86_avx512_vpdpbusds_128; in UpgradeIntrinsicCall()
4037 IID = Intrinsic::x86_avx512_vpdpbusds_256; in UpgradeIntrinsicCall()
4039 IID = Intrinsic::x86_avx512_vpdpbusds_512; in UpgradeIntrinsicCall()
4045 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in UpgradeIntrinsicCall()
4057 Intrinsic::ID IID; in UpgradeIntrinsicCall() local
4059 IID = Intrinsic::x86_avx512_vpdpwssd_128; in UpgradeIntrinsicCall()
4061 IID = Intrinsic::x86_avx512_vpdpwssd_256; in UpgradeIntrinsicCall()
4063 IID = Intrinsic::x86_avx512_vpdpwssd_512; in UpgradeIntrinsicCall()
4065 IID = Intrinsic::x86_avx512_vpdpwssds_128; in UpgradeIntrinsicCall()
4067 IID = Intrinsic::x86_avx512_vpdpwssds_256; in UpgradeIntrinsicCall()
4069 IID = Intrinsic::x86_avx512_vpdpwssds_512; in UpgradeIntrinsicCall()
4075 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), in UpgradeIntrinsicCall()
4083 Intrinsic::ID IID; in UpgradeIntrinsicCall() local
4085 IID = Intrinsic::x86_addcarry_32; in UpgradeIntrinsicCall()
4087 IID = Intrinsic::x86_addcarry_64; in UpgradeIntrinsicCall()
4089 IID = Intrinsic::x86_subborrow_32; in UpgradeIntrinsicCall()
4091 IID = Intrinsic::x86_subborrow_64; in UpgradeIntrinsicCall()
4099 Intrinsic::getDeclaration(CI->getModule(), IID), in UpgradeIntrinsicCall()
4171 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name); in UpgradeIntrinsicCall() local
4172 if (IID != Intrinsic::not_intrinsic && in UpgradeIntrinsicCall()
4175 NewFn = Intrinsic::getDeclaration(F->getParent(), IID); in UpgradeIntrinsicCall()