1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py 2; RUN: llc < %s -mtriple=x86_64--linux-gnu -mcpu=x86-64 -mattr=+sse2 | FileCheck %s --check-prefix=CHECK --check-prefix=SSE 3; RUN: llc < %s -mtriple=x86_64--linux-gnu -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1 4; RUN: llc < %s -mtriple=x86_64--linux-gnu -mcpu=x86-64 -mattr=+avx512f | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512 5 6declare double @__sqrt_finite(double) 7declare float @__sqrtf_finite(float) 8declare x86_fp80 @__sqrtl_finite(x86_fp80) 9declare float @llvm.sqrt.f32(float) 10declare <4 x float> @llvm.sqrt.v4f32(<4 x float>) 11declare <8 x float> @llvm.sqrt.v8f32(<8 x float>) 12declare <16 x float> @llvm.sqrt.v16f32(<16 x float>) 13declare double @llvm.sqrt.f64(double) 14declare <2 x double> @llvm.sqrt.v2f64(<2 x double>) 15 16declare float @llvm.fabs.f32(float) 17declare <4 x float> @llvm.fabs.v4f32(<4 x float>) 18declare double @llvm.fabs.f64(double) 19 20define double @finite_f64_no_estimate(double %d) #0 { 21; SSE-LABEL: finite_f64_no_estimate: 22; SSE: # %bb.0: 23; SSE-NEXT: sqrtsd %xmm0, %xmm0 24; SSE-NEXT: retq 25; 26; AVX-LABEL: finite_f64_no_estimate: 27; AVX: # %bb.0: 28; AVX-NEXT: vsqrtsd %xmm0, %xmm0, %xmm0 29; AVX-NEXT: retq 30 %call = tail call double @__sqrt_finite(double %d) #2 31 ret double %call 32} 33 34; No estimates for doubles. 35 36define double @finite_f64_estimate(double %d) #1 { 37; SSE-LABEL: finite_f64_estimate: 38; SSE: # %bb.0: 39; SSE-NEXT: sqrtsd %xmm0, %xmm0 40; SSE-NEXT: retq 41; 42; AVX-LABEL: finite_f64_estimate: 43; AVX: # %bb.0: 44; AVX-NEXT: vsqrtsd %xmm0, %xmm0, %xmm0 45; AVX-NEXT: retq 46 %call = tail call double @__sqrt_finite(double %d) #2 47 ret double %call 48} 49 50define float @finite_f32_no_estimate(float %f) #0 { 51; SSE-LABEL: finite_f32_no_estimate: 52; SSE: # %bb.0: 53; SSE-NEXT: sqrtss %xmm0, %xmm0 54; SSE-NEXT: retq 55; 56; AVX-LABEL: finite_f32_no_estimate: 57; AVX: # %bb.0: 58; AVX-NEXT: vsqrtss %xmm0, %xmm0, %xmm0 59; AVX-NEXT: retq 60 %call = tail call float @__sqrtf_finite(float %f) #2 61 ret float %call 62} 63 64define float @finite_f32_estimate_ieee(float %f) #1 { 65; SSE-LABEL: finite_f32_estimate_ieee: 66; SSE: # %bb.0: 67; SSE-NEXT: sqrtss %xmm0, %xmm0 68; SSE-NEXT: retq 69; 70; AVX-LABEL: finite_f32_estimate_ieee: 71; AVX: # %bb.0: 72; AVX-NEXT: vsqrtss %xmm0, %xmm0, %xmm0 73; AVX-NEXT: retq 74 %call = tail call float @__sqrtf_finite(float %f) #2 75 ret float %call 76} 77 78define float @finite_f32_estimate_ieee_ninf(float %f) #1 { 79; SSE-LABEL: finite_f32_estimate_ieee_ninf: 80; SSE: # %bb.0: 81; SSE-NEXT: rsqrtss %xmm0, %xmm1 82; SSE-NEXT: movaps %xmm0, %xmm2 83; SSE-NEXT: mulss %xmm1, %xmm2 84; SSE-NEXT: movss {{.*#+}} xmm3 = mem[0],zero,zero,zero 85; SSE-NEXT: mulss %xmm2, %xmm3 86; SSE-NEXT: mulss %xmm1, %xmm2 87; SSE-NEXT: addss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 88; SSE-NEXT: mulss %xmm3, %xmm2 89; SSE-NEXT: andps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 90; SSE-NEXT: cmpltss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 91; SSE-NEXT: andnps %xmm2, %xmm0 92; SSE-NEXT: retq 93; 94; AVX1-LABEL: finite_f32_estimate_ieee_ninf: 95; AVX1: # %bb.0: 96; AVX1-NEXT: vrsqrtss %xmm0, %xmm0, %xmm1 97; AVX1-NEXT: vmulss %xmm1, %xmm0, %xmm2 98; AVX1-NEXT: vmulss %xmm1, %xmm2, %xmm1 99; AVX1-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 100; AVX1-NEXT: vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 101; AVX1-NEXT: vmulss %xmm1, %xmm2, %xmm1 102; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 103; AVX1-NEXT: vcmpltss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 104; AVX1-NEXT: vandnps %xmm1, %xmm0, %xmm0 105; AVX1-NEXT: retq 106; 107; AVX512-LABEL: finite_f32_estimate_ieee_ninf: 108; AVX512: # %bb.0: 109; AVX512-NEXT: vrsqrtss %xmm0, %xmm0, %xmm1 110; AVX512-NEXT: vbroadcastss {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN] 111; AVX512-NEXT: vandps %xmm2, %xmm0, %xmm2 112; AVX512-NEXT: vmulss %xmm1, %xmm0, %xmm0 113; AVX512-NEXT: vfmadd213ss {{.*#+}} xmm1 = (xmm0 * xmm1) + mem 114; AVX512-NEXT: vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 115; AVX512-NEXT: vcmpltss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %k1 116; AVX512-NEXT: vmulss %xmm1, %xmm0, %xmm0 117; AVX512-NEXT: vxorps %xmm1, %xmm1, %xmm1 118; AVX512-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1} 119; AVX512-NEXT: retq 120 %call = tail call ninf afn float @__sqrtf_finite(float %f) #2 121 ret float %call 122} 123 124define float @finite_f32_estimate_daz(float %f) #4 { 125; SSE-LABEL: finite_f32_estimate_daz: 126; SSE: # %bb.0: 127; SSE-NEXT: sqrtss %xmm0, %xmm0 128; SSE-NEXT: retq 129; 130; AVX-LABEL: finite_f32_estimate_daz: 131; AVX: # %bb.0: 132; AVX-NEXT: vsqrtss %xmm0, %xmm0, %xmm0 133; AVX-NEXT: retq 134 %call = tail call float @__sqrtf_finite(float %f) #2 135 ret float %call 136} 137 138define float @finite_f32_estimate_daz_ninf(float %f) #4 { 139; SSE-LABEL: finite_f32_estimate_daz_ninf: 140; SSE: # %bb.0: 141; SSE-NEXT: rsqrtss %xmm0, %xmm1 142; SSE-NEXT: movaps %xmm0, %xmm2 143; SSE-NEXT: mulss %xmm1, %xmm2 144; SSE-NEXT: movss {{.*#+}} xmm3 = mem[0],zero,zero,zero 145; SSE-NEXT: mulss %xmm2, %xmm3 146; SSE-NEXT: mulss %xmm1, %xmm2 147; SSE-NEXT: addss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 148; SSE-NEXT: mulss %xmm3, %xmm2 149; SSE-NEXT: xorps %xmm1, %xmm1 150; SSE-NEXT: cmpeqss %xmm1, %xmm0 151; SSE-NEXT: andnps %xmm2, %xmm0 152; SSE-NEXT: retq 153; 154; AVX1-LABEL: finite_f32_estimate_daz_ninf: 155; AVX1: # %bb.0: 156; AVX1-NEXT: vrsqrtss %xmm0, %xmm0, %xmm1 157; AVX1-NEXT: vmulss %xmm1, %xmm0, %xmm2 158; AVX1-NEXT: vmulss %xmm1, %xmm2, %xmm1 159; AVX1-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 160; AVX1-NEXT: vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 161; AVX1-NEXT: vmulss %xmm1, %xmm2, %xmm1 162; AVX1-NEXT: vxorps %xmm2, %xmm2, %xmm2 163; AVX1-NEXT: vcmpeqss %xmm2, %xmm0, %xmm0 164; AVX1-NEXT: vandnps %xmm1, %xmm0, %xmm0 165; AVX1-NEXT: retq 166; 167; AVX512-LABEL: finite_f32_estimate_daz_ninf: 168; AVX512: # %bb.0: 169; AVX512-NEXT: vrsqrtss %xmm0, %xmm0, %xmm1 170; AVX512-NEXT: vmulss %xmm1, %xmm0, %xmm2 171; AVX512-NEXT: vfmadd213ss {{.*#+}} xmm1 = (xmm2 * xmm1) + mem 172; AVX512-NEXT: vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 173; AVX512-NEXT: vmulss %xmm1, %xmm2, %xmm1 174; AVX512-NEXT: vxorps %xmm2, %xmm2, %xmm2 175; AVX512-NEXT: vcmpeqss %xmm2, %xmm0, %k1 176; AVX512-NEXT: vmovss %xmm2, %xmm1, %xmm1 {%k1} 177; AVX512-NEXT: vmovaps %xmm1, %xmm0 178; AVX512-NEXT: retq 179 %call = tail call ninf afn float @__sqrtf_finite(float %f) #2 180 ret float %call 181} 182 183define x86_fp80 @finite_f80_no_estimate(x86_fp80 %ld) #0 { 184; CHECK-LABEL: finite_f80_no_estimate: 185; CHECK: # %bb.0: 186; CHECK-NEXT: fldt {{[0-9]+}}(%rsp) 187; CHECK-NEXT: fsqrt 188; CHECK-NEXT: retq 189 %call = tail call x86_fp80 @__sqrtl_finite(x86_fp80 %ld) #2 190 ret x86_fp80 %call 191} 192 193; Don't die on the impossible. 194 195define x86_fp80 @finite_f80_estimate_but_no(x86_fp80 %ld) #1 { 196; CHECK-LABEL: finite_f80_estimate_but_no: 197; CHECK: # %bb.0: 198; CHECK-NEXT: fldt {{[0-9]+}}(%rsp) 199; CHECK-NEXT: fsqrt 200; CHECK-NEXT: retq 201 %call = tail call x86_fp80 @__sqrtl_finite(x86_fp80 %ld) #2 202 ret x86_fp80 %call 203} 204 205; PR34994 - https://bugs.llvm.org/show_bug.cgi?id=34994 206 207define float @sqrtf_check_denorms(float %x) #3 { 208; SSE-LABEL: sqrtf_check_denorms: 209; SSE: # %bb.0: 210; SSE-NEXT: sqrtss %xmm0, %xmm0 211; SSE-NEXT: retq 212; 213; AVX-LABEL: sqrtf_check_denorms: 214; AVX: # %bb.0: 215; AVX-NEXT: vsqrtss %xmm0, %xmm0, %xmm0 216; AVX-NEXT: retq 217 %call = tail call float @__sqrtf_finite(float %x) #2 218 ret float %call 219} 220 221define float @sqrtf_check_denorms_ninf(float %x) #3 { 222; SSE-LABEL: sqrtf_check_denorms_ninf: 223; SSE: # %bb.0: 224; SSE-NEXT: rsqrtss %xmm0, %xmm1 225; SSE-NEXT: movaps %xmm0, %xmm2 226; SSE-NEXT: mulss %xmm1, %xmm2 227; SSE-NEXT: movss {{.*#+}} xmm3 = mem[0],zero,zero,zero 228; SSE-NEXT: mulss %xmm2, %xmm3 229; SSE-NEXT: mulss %xmm1, %xmm2 230; SSE-NEXT: addss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 231; SSE-NEXT: mulss %xmm3, %xmm2 232; SSE-NEXT: andps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 233; SSE-NEXT: cmpltss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 234; SSE-NEXT: andnps %xmm2, %xmm0 235; SSE-NEXT: retq 236; 237; AVX1-LABEL: sqrtf_check_denorms_ninf: 238; AVX1: # %bb.0: 239; AVX1-NEXT: vrsqrtss %xmm0, %xmm0, %xmm1 240; AVX1-NEXT: vmulss %xmm1, %xmm0, %xmm2 241; AVX1-NEXT: vmulss %xmm1, %xmm2, %xmm1 242; AVX1-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 243; AVX1-NEXT: vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 244; AVX1-NEXT: vmulss %xmm1, %xmm2, %xmm1 245; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 246; AVX1-NEXT: vcmpltss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 247; AVX1-NEXT: vandnps %xmm1, %xmm0, %xmm0 248; AVX1-NEXT: retq 249; 250; AVX512-LABEL: sqrtf_check_denorms_ninf: 251; AVX512: # %bb.0: 252; AVX512-NEXT: vrsqrtss %xmm0, %xmm0, %xmm1 253; AVX512-NEXT: vbroadcastss {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN] 254; AVX512-NEXT: vandps %xmm2, %xmm0, %xmm2 255; AVX512-NEXT: vmulss %xmm1, %xmm0, %xmm0 256; AVX512-NEXT: vfmadd213ss {{.*#+}} xmm1 = (xmm0 * xmm1) + mem 257; AVX512-NEXT: vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 258; AVX512-NEXT: vcmpltss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %k1 259; AVX512-NEXT: vmulss %xmm1, %xmm0, %xmm0 260; AVX512-NEXT: vxorps %xmm1, %xmm1, %xmm1 261; AVX512-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1} 262; AVX512-NEXT: retq 263 %call = tail call ninf afn float @__sqrtf_finite(float %x) #2 264 ret float %call 265} 266 267define <4 x float> @sqrt_v4f32_check_denorms(<4 x float> %x) #3 { 268; SSE-LABEL: sqrt_v4f32_check_denorms: 269; SSE: # %bb.0: 270; SSE-NEXT: sqrtps %xmm0, %xmm0 271; SSE-NEXT: retq 272; 273; AVX-LABEL: sqrt_v4f32_check_denorms: 274; AVX: # %bb.0: 275; AVX-NEXT: vsqrtps %xmm0, %xmm0 276; AVX-NEXT: retq 277 %call = tail call <4 x float> @llvm.sqrt.v4f32(<4 x float> %x) #2 278 ret <4 x float> %call 279} 280 281define <4 x float> @sqrt_v4f32_check_denorms_ninf(<4 x float> %x) #3 { 282; SSE-LABEL: sqrt_v4f32_check_denorms_ninf: 283; SSE: # %bb.0: 284; SSE-NEXT: rsqrtps %xmm0, %xmm1 285; SSE-NEXT: movaps %xmm0, %xmm2 286; SSE-NEXT: mulps %xmm1, %xmm2 287; SSE-NEXT: movaps {{.*#+}} xmm3 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1] 288; SSE-NEXT: mulps %xmm2, %xmm3 289; SSE-NEXT: mulps %xmm1, %xmm2 290; SSE-NEXT: addps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 291; SSE-NEXT: mulps %xmm3, %xmm2 292; SSE-NEXT: andps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 293; SSE-NEXT: movaps {{.*#+}} xmm1 = [1.17549435E-38,1.17549435E-38,1.17549435E-38,1.17549435E-38] 294; SSE-NEXT: cmpleps %xmm0, %xmm1 295; SSE-NEXT: andps %xmm2, %xmm1 296; SSE-NEXT: movaps %xmm1, %xmm0 297; SSE-NEXT: retq 298; 299; AVX1-LABEL: sqrt_v4f32_check_denorms_ninf: 300; AVX1: # %bb.0: 301; AVX1-NEXT: vrsqrtps %xmm0, %xmm1 302; AVX1-NEXT: vmulps %xmm1, %xmm0, %xmm2 303; AVX1-NEXT: vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm3 304; AVX1-NEXT: vmulps %xmm1, %xmm2, %xmm1 305; AVX1-NEXT: vaddps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 306; AVX1-NEXT: vmulps %xmm1, %xmm3, %xmm1 307; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 308; AVX1-NEXT: vmovaps {{.*#+}} xmm2 = [1.17549435E-38,1.17549435E-38,1.17549435E-38,1.17549435E-38] 309; AVX1-NEXT: vcmpleps %xmm0, %xmm2, %xmm0 310; AVX1-NEXT: vandps %xmm1, %xmm0, %xmm0 311; AVX1-NEXT: retq 312; 313; AVX512-LABEL: sqrt_v4f32_check_denorms_ninf: 314; AVX512: # %bb.0: 315; AVX512-NEXT: vrsqrtps %xmm0, %xmm1 316; AVX512-NEXT: vmulps %xmm1, %xmm0, %xmm2 317; AVX512-NEXT: vbroadcastss {{.*#+}} xmm3 = [-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0] 318; AVX512-NEXT: vfmadd231ps {{.*#+}} xmm3 = (xmm2 * xmm1) + xmm3 319; AVX512-NEXT: vbroadcastss {{.*#+}} xmm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1] 320; AVX512-NEXT: vmulps %xmm1, %xmm2, %xmm1 321; AVX512-NEXT: vmulps %xmm3, %xmm1, %xmm1 322; AVX512-NEXT: vbroadcastss {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN] 323; AVX512-NEXT: vandps %xmm2, %xmm0, %xmm0 324; AVX512-NEXT: vbroadcastss {{.*#+}} xmm2 = [1.17549435E-38,1.17549435E-38,1.17549435E-38,1.17549435E-38] 325; AVX512-NEXT: vcmpleps %xmm0, %xmm2, %xmm0 326; AVX512-NEXT: vandps %xmm1, %xmm0, %xmm0 327; AVX512-NEXT: retq 328 %call = tail call ninf afn <4 x float> @llvm.sqrt.v4f32(<4 x float> %x) #2 329 ret <4 x float> %call 330} 331 332define float @f32_no_estimate(float %x) #0 { 333; SSE-LABEL: f32_no_estimate: 334; SSE: # %bb.0: 335; SSE-NEXT: sqrtss %xmm0, %xmm1 336; SSE-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero 337; SSE-NEXT: divss %xmm1, %xmm0 338; SSE-NEXT: retq 339; 340; AVX-LABEL: f32_no_estimate: 341; AVX: # %bb.0: 342; AVX-NEXT: vsqrtss %xmm0, %xmm0, %xmm0 343; AVX-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero 344; AVX-NEXT: vdivss %xmm0, %xmm1, %xmm0 345; AVX-NEXT: retq 346 %sqrt = tail call float @llvm.sqrt.f32(float %x) 347 %div = fdiv fast float 1.0, %sqrt 348 ret float %div 349} 350 351define float @f32_estimate(float %x) #1 { 352; SSE-LABEL: f32_estimate: 353; SSE: # %bb.0: 354; SSE-NEXT: rsqrtss %xmm0, %xmm1 355; SSE-NEXT: mulss %xmm1, %xmm0 356; SSE-NEXT: mulss %xmm1, %xmm0 357; SSE-NEXT: addss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 358; SSE-NEXT: mulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 359; SSE-NEXT: mulss %xmm1, %xmm0 360; SSE-NEXT: retq 361; 362; AVX1-LABEL: f32_estimate: 363; AVX1: # %bb.0: 364; AVX1-NEXT: vrsqrtss %xmm0, %xmm0, %xmm1 365; AVX1-NEXT: vmulss %xmm1, %xmm0, %xmm0 366; AVX1-NEXT: vmulss %xmm1, %xmm0, %xmm0 367; AVX1-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 368; AVX1-NEXT: vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 369; AVX1-NEXT: vmulss %xmm0, %xmm1, %xmm0 370; AVX1-NEXT: retq 371; 372; AVX512-LABEL: f32_estimate: 373; AVX512: # %bb.0: 374; AVX512-NEXT: vrsqrtss %xmm0, %xmm0, %xmm1 375; AVX512-NEXT: vmulss %xmm1, %xmm0, %xmm0 376; AVX512-NEXT: vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + mem 377; AVX512-NEXT: vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 378; AVX512-NEXT: vmulss %xmm0, %xmm1, %xmm0 379; AVX512-NEXT: retq 380 %sqrt = tail call float @llvm.sqrt.f32(float %x) 381 %div = fdiv fast float 1.0, %sqrt 382 ret float %div 383} 384 385define float @f32_estimate2(float %x) #5 { 386; SSE-LABEL: f32_estimate2: 387; SSE: # %bb.0: 388; SSE-NEXT: rsqrtss %xmm0, %xmm1 389; SSE-NEXT: mulss %xmm0, %xmm1 390; SSE-NEXT: andps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 391; SSE-NEXT: cmpltss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 392; SSE-NEXT: andnps %xmm1, %xmm0 393; SSE-NEXT: retq 394; 395; AVX1-LABEL: f32_estimate2: 396; AVX1: # %bb.0: 397; AVX1-NEXT: vrsqrtss %xmm0, %xmm0, %xmm1 398; AVX1-NEXT: vmulss %xmm1, %xmm0, %xmm1 399; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 400; AVX1-NEXT: vcmpltss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 401; AVX1-NEXT: vandnps %xmm1, %xmm0, %xmm0 402; AVX1-NEXT: retq 403; 404; AVX512-LABEL: f32_estimate2: 405; AVX512: # %bb.0: 406; AVX512-NEXT: vbroadcastss {{.*#+}} xmm1 = [NaN,NaN,NaN,NaN] 407; AVX512-NEXT: vrsqrtss %xmm0, %xmm0, %xmm2 408; AVX512-NEXT: vandps %xmm1, %xmm0, %xmm1 409; AVX512-NEXT: vcmpltss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %k1 410; AVX512-NEXT: vmulss %xmm2, %xmm0, %xmm0 411; AVX512-NEXT: vxorps %xmm1, %xmm1, %xmm1 412; AVX512-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1} 413; AVX512-NEXT: retq 414 %sqrt = tail call fast float @llvm.sqrt.f32(float %x) 415 ret float %sqrt 416} 417 418define <4 x float> @v4f32_no_estimate(<4 x float> %x) #0 { 419; SSE-LABEL: v4f32_no_estimate: 420; SSE: # %bb.0: 421; SSE-NEXT: sqrtps %xmm0, %xmm1 422; SSE-NEXT: movaps {{.*#+}} xmm0 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] 423; SSE-NEXT: divps %xmm1, %xmm0 424; SSE-NEXT: retq 425; 426; AVX1-LABEL: v4f32_no_estimate: 427; AVX1: # %bb.0: 428; AVX1-NEXT: vsqrtps %xmm0, %xmm0 429; AVX1-NEXT: vmovaps {{.*#+}} xmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] 430; AVX1-NEXT: vdivps %xmm0, %xmm1, %xmm0 431; AVX1-NEXT: retq 432; 433; AVX512-LABEL: v4f32_no_estimate: 434; AVX512: # %bb.0: 435; AVX512-NEXT: vsqrtps %xmm0, %xmm0 436; AVX512-NEXT: vbroadcastss {{.*#+}} xmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] 437; AVX512-NEXT: vdivps %xmm0, %xmm1, %xmm0 438; AVX512-NEXT: retq 439 %sqrt = tail call <4 x float> @llvm.sqrt.v4f32(<4 x float> %x) 440 %div = fdiv fast <4 x float> <float 1.0, float 1.0, float 1.0, float 1.0>, %sqrt 441 ret <4 x float> %div 442} 443 444define <4 x float> @v4f32_estimate(<4 x float> %x) #1 { 445; SSE-LABEL: v4f32_estimate: 446; SSE: # %bb.0: 447; SSE-NEXT: rsqrtps %xmm0, %xmm1 448; SSE-NEXT: mulps %xmm1, %xmm0 449; SSE-NEXT: mulps %xmm1, %xmm0 450; SSE-NEXT: addps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 451; SSE-NEXT: mulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 452; SSE-NEXT: mulps %xmm1, %xmm0 453; SSE-NEXT: retq 454; 455; AVX1-LABEL: v4f32_estimate: 456; AVX1: # %bb.0: 457; AVX1-NEXT: vrsqrtps %xmm0, %xmm1 458; AVX1-NEXT: vmulps %xmm1, %xmm0, %xmm0 459; AVX1-NEXT: vmulps %xmm1, %xmm0, %xmm0 460; AVX1-NEXT: vaddps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 461; AVX1-NEXT: vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 462; AVX1-NEXT: vmulps %xmm0, %xmm1, %xmm0 463; AVX1-NEXT: retq 464; 465; AVX512-LABEL: v4f32_estimate: 466; AVX512: # %bb.0: 467; AVX512-NEXT: vrsqrtps %xmm0, %xmm1 468; AVX512-NEXT: vmulps %xmm1, %xmm0, %xmm0 469; AVX512-NEXT: vbroadcastss {{.*#+}} xmm2 = [-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0] 470; AVX512-NEXT: vfmadd231ps {{.*#+}} xmm2 = (xmm1 * xmm0) + xmm2 471; AVX512-NEXT: vbroadcastss {{.*#+}} xmm0 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1] 472; AVX512-NEXT: vmulps %xmm0, %xmm1, %xmm0 473; AVX512-NEXT: vmulps %xmm2, %xmm0, %xmm0 474; AVX512-NEXT: retq 475 %sqrt = tail call <4 x float> @llvm.sqrt.v4f32(<4 x float> %x) 476 %div = fdiv fast <4 x float> <float 1.0, float 1.0, float 1.0, float 1.0>, %sqrt 477 ret <4 x float> %div 478} 479 480define <4 x float> @v4f32_estimate2(<4 x float> %x) #5 { 481; SSE-LABEL: v4f32_estimate2: 482; SSE: # %bb.0: 483; SSE-NEXT: rsqrtps %xmm0, %xmm2 484; SSE-NEXT: mulps %xmm0, %xmm2 485; SSE-NEXT: andps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 486; SSE-NEXT: movaps {{.*#+}} xmm1 = [1.17549435E-38,1.17549435E-38,1.17549435E-38,1.17549435E-38] 487; SSE-NEXT: cmpleps %xmm0, %xmm1 488; SSE-NEXT: andps %xmm2, %xmm1 489; SSE-NEXT: movaps %xmm1, %xmm0 490; SSE-NEXT: retq 491; 492; AVX1-LABEL: v4f32_estimate2: 493; AVX1: # %bb.0: 494; AVX1-NEXT: vrsqrtps %xmm0, %xmm1 495; AVX1-NEXT: vmulps %xmm1, %xmm0, %xmm1 496; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 497; AVX1-NEXT: vmovaps {{.*#+}} xmm2 = [1.17549435E-38,1.17549435E-38,1.17549435E-38,1.17549435E-38] 498; AVX1-NEXT: vcmpleps %xmm0, %xmm2, %xmm0 499; AVX1-NEXT: vandps %xmm1, %xmm0, %xmm0 500; AVX1-NEXT: retq 501; 502; AVX512-LABEL: v4f32_estimate2: 503; AVX512: # %bb.0: 504; AVX512-NEXT: vrsqrtps %xmm0, %xmm1 505; AVX512-NEXT: vmulps %xmm1, %xmm0, %xmm1 506; AVX512-NEXT: vbroadcastss {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN] 507; AVX512-NEXT: vandps %xmm2, %xmm0, %xmm0 508; AVX512-NEXT: vbroadcastss {{.*#+}} xmm2 = [1.17549435E-38,1.17549435E-38,1.17549435E-38,1.17549435E-38] 509; AVX512-NEXT: vcmpleps %xmm0, %xmm2, %xmm0 510; AVX512-NEXT: vandps %xmm1, %xmm0, %xmm0 511; AVX512-NEXT: retq 512 %sqrt = tail call fast <4 x float> @llvm.sqrt.v4f32(<4 x float> %x) 513 ret <4 x float> %sqrt 514} 515 516define <8 x float> @v8f32_no_estimate(<8 x float> %x) #0 { 517; SSE-LABEL: v8f32_no_estimate: 518; SSE: # %bb.0: 519; SSE-NEXT: sqrtps %xmm1, %xmm2 520; SSE-NEXT: sqrtps %xmm0, %xmm3 521; SSE-NEXT: movaps {{.*#+}} xmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] 522; SSE-NEXT: movaps %xmm1, %xmm0 523; SSE-NEXT: divps %xmm3, %xmm0 524; SSE-NEXT: divps %xmm2, %xmm1 525; SSE-NEXT: retq 526; 527; AVX1-LABEL: v8f32_no_estimate: 528; AVX1: # %bb.0: 529; AVX1-NEXT: vsqrtps %ymm0, %ymm0 530; AVX1-NEXT: vmovaps {{.*#+}} ymm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] 531; AVX1-NEXT: vdivps %ymm0, %ymm1, %ymm0 532; AVX1-NEXT: retq 533; 534; AVX512-LABEL: v8f32_no_estimate: 535; AVX512: # %bb.0: 536; AVX512-NEXT: vsqrtps %ymm0, %ymm0 537; AVX512-NEXT: vbroadcastss {{.*#+}} ymm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] 538; AVX512-NEXT: vdivps %ymm0, %ymm1, %ymm0 539; AVX512-NEXT: retq 540 %sqrt = tail call <8 x float> @llvm.sqrt.v8f32(<8 x float> %x) 541 %div = fdiv fast <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, %sqrt 542 ret <8 x float> %div 543} 544 545define <8 x float> @v8f32_estimate(<8 x float> %x) #1 { 546; SSE-LABEL: v8f32_estimate: 547; SSE: # %bb.0: 548; SSE-NEXT: rsqrtps %xmm0, %xmm2 549; SSE-NEXT: movaps {{.*#+}} xmm3 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1] 550; SSE-NEXT: mulps %xmm2, %xmm0 551; SSE-NEXT: mulps %xmm2, %xmm0 552; SSE-NEXT: mulps %xmm3, %xmm2 553; SSE-NEXT: movaps {{.*#+}} xmm4 = [-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0] 554; SSE-NEXT: addps %xmm4, %xmm0 555; SSE-NEXT: mulps %xmm2, %xmm0 556; SSE-NEXT: rsqrtps %xmm1, %xmm2 557; SSE-NEXT: mulps %xmm2, %xmm3 558; SSE-NEXT: mulps %xmm2, %xmm1 559; SSE-NEXT: mulps %xmm2, %xmm1 560; SSE-NEXT: addps %xmm4, %xmm1 561; SSE-NEXT: mulps %xmm3, %xmm1 562; SSE-NEXT: retq 563; 564; AVX1-LABEL: v8f32_estimate: 565; AVX1: # %bb.0: 566; AVX1-NEXT: vrsqrtps %ymm0, %ymm1 567; AVX1-NEXT: vmulps %ymm1, %ymm0, %ymm0 568; AVX1-NEXT: vmulps %ymm1, %ymm0, %ymm0 569; AVX1-NEXT: vaddps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 570; AVX1-NEXT: vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 571; AVX1-NEXT: vmulps %ymm0, %ymm1, %ymm0 572; AVX1-NEXT: retq 573; 574; AVX512-LABEL: v8f32_estimate: 575; AVX512: # %bb.0: 576; AVX512-NEXT: vrsqrtps %ymm0, %ymm1 577; AVX512-NEXT: vmulps %ymm1, %ymm0, %ymm0 578; AVX512-NEXT: vbroadcastss {{.*#+}} ymm2 = [-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0] 579; AVX512-NEXT: vfmadd231ps {{.*#+}} ymm2 = (ymm1 * ymm0) + ymm2 580; AVX512-NEXT: vbroadcastss {{.*#+}} ymm0 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1] 581; AVX512-NEXT: vmulps %ymm0, %ymm1, %ymm0 582; AVX512-NEXT: vmulps %ymm2, %ymm0, %ymm0 583; AVX512-NEXT: retq 584 %sqrt = tail call <8 x float> @llvm.sqrt.v8f32(<8 x float> %x) 585 %div = fdiv fast <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, %sqrt 586 ret <8 x float> %div 587} 588 589define <16 x float> @v16f32_no_estimate(<16 x float> %x) #0 { 590; SSE-LABEL: v16f32_no_estimate: 591; SSE: # %bb.0: 592; SSE-NEXT: sqrtps %xmm3, %xmm4 593; SSE-NEXT: sqrtps %xmm2, %xmm5 594; SSE-NEXT: sqrtps %xmm1, %xmm2 595; SSE-NEXT: sqrtps %xmm0, %xmm1 596; SSE-NEXT: movaps {{.*#+}} xmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] 597; SSE-NEXT: movaps %xmm3, %xmm0 598; SSE-NEXT: divps %xmm1, %xmm0 599; SSE-NEXT: movaps %xmm3, %xmm1 600; SSE-NEXT: divps %xmm2, %xmm1 601; SSE-NEXT: movaps %xmm3, %xmm2 602; SSE-NEXT: divps %xmm5, %xmm2 603; SSE-NEXT: divps %xmm4, %xmm3 604; SSE-NEXT: retq 605; 606; AVX1-LABEL: v16f32_no_estimate: 607; AVX1: # %bb.0: 608; AVX1-NEXT: vsqrtps %ymm1, %ymm1 609; AVX1-NEXT: vsqrtps %ymm0, %ymm0 610; AVX1-NEXT: vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] 611; AVX1-NEXT: vdivps %ymm0, %ymm2, %ymm0 612; AVX1-NEXT: vdivps %ymm1, %ymm2, %ymm1 613; AVX1-NEXT: retq 614; 615; AVX512-LABEL: v16f32_no_estimate: 616; AVX512: # %bb.0: 617; AVX512-NEXT: vsqrtps %zmm0, %zmm0 618; AVX512-NEXT: vbroadcastss {{.*#+}} zmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] 619; AVX512-NEXT: vdivps %zmm0, %zmm1, %zmm0 620; AVX512-NEXT: retq 621 %sqrt = tail call <16 x float> @llvm.sqrt.v16f32(<16 x float> %x) 622 %div = fdiv fast <16 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, %sqrt 623 ret <16 x float> %div 624} 625 626define <16 x float> @v16f32_estimate(<16 x float> %x) #1 { 627; SSE-LABEL: v16f32_estimate: 628; SSE: # %bb.0: 629; SSE-NEXT: rsqrtps %xmm0, %xmm5 630; SSE-NEXT: movaps {{.*#+}} xmm4 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1] 631; SSE-NEXT: mulps %xmm5, %xmm0 632; SSE-NEXT: mulps %xmm5, %xmm0 633; SSE-NEXT: movaps %xmm5, %xmm6 634; SSE-NEXT: mulps %xmm4, %xmm6 635; SSE-NEXT: movaps {{.*#+}} xmm5 = [-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0] 636; SSE-NEXT: addps %xmm5, %xmm0 637; SSE-NEXT: mulps %xmm6, %xmm0 638; SSE-NEXT: rsqrtps %xmm1, %xmm6 639; SSE-NEXT: mulps %xmm6, %xmm1 640; SSE-NEXT: mulps %xmm6, %xmm1 641; SSE-NEXT: mulps %xmm4, %xmm6 642; SSE-NEXT: addps %xmm5, %xmm1 643; SSE-NEXT: mulps %xmm6, %xmm1 644; SSE-NEXT: rsqrtps %xmm2, %xmm6 645; SSE-NEXT: mulps %xmm6, %xmm2 646; SSE-NEXT: mulps %xmm6, %xmm2 647; SSE-NEXT: mulps %xmm4, %xmm6 648; SSE-NEXT: addps %xmm5, %xmm2 649; SSE-NEXT: mulps %xmm6, %xmm2 650; SSE-NEXT: rsqrtps %xmm3, %xmm6 651; SSE-NEXT: mulps %xmm6, %xmm4 652; SSE-NEXT: mulps %xmm6, %xmm3 653; SSE-NEXT: mulps %xmm6, %xmm3 654; SSE-NEXT: addps %xmm5, %xmm3 655; SSE-NEXT: mulps %xmm4, %xmm3 656; SSE-NEXT: retq 657; 658; AVX1-LABEL: v16f32_estimate: 659; AVX1: # %bb.0: 660; AVX1-NEXT: vrsqrtps %ymm0, %ymm2 661; AVX1-NEXT: vmovaps {{.*#+}} ymm3 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1] 662; AVX1-NEXT: vmulps %ymm3, %ymm2, %ymm4 663; AVX1-NEXT: vmulps %ymm2, %ymm0, %ymm0 664; AVX1-NEXT: vmulps %ymm2, %ymm0, %ymm0 665; AVX1-NEXT: vmovaps {{.*#+}} ymm2 = [-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0] 666; AVX1-NEXT: vaddps %ymm2, %ymm0, %ymm0 667; AVX1-NEXT: vrsqrtps %ymm1, %ymm5 668; AVX1-NEXT: vmulps %ymm0, %ymm4, %ymm0 669; AVX1-NEXT: vmulps %ymm3, %ymm5, %ymm3 670; AVX1-NEXT: vmulps %ymm5, %ymm1, %ymm1 671; AVX1-NEXT: vmulps %ymm5, %ymm1, %ymm1 672; AVX1-NEXT: vaddps %ymm2, %ymm1, %ymm1 673; AVX1-NEXT: vmulps %ymm1, %ymm3, %ymm1 674; AVX1-NEXT: retq 675; 676; AVX512-LABEL: v16f32_estimate: 677; AVX512: # %bb.0: 678; AVX512-NEXT: vrsqrt14ps %zmm0, %zmm1 679; AVX512-NEXT: vmulps %zmm1, %zmm0, %zmm0 680; AVX512-NEXT: vfmadd213ps {{.*#+}} zmm0 = (zmm1 * zmm0) + mem 681; AVX512-NEXT: vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1 682; AVX512-NEXT: vmulps %zmm0, %zmm1, %zmm0 683; AVX512-NEXT: retq 684 %sqrt = tail call <16 x float> @llvm.sqrt.v16f32(<16 x float> %x) 685 %div = fdiv fast <16 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, %sqrt 686 ret <16 x float> %div 687} 688 689; x / (fabs(y) * sqrt(z)) --> x * rsqrt(y*y*z) 690 691define float @div_sqrt_fabs_f32(float %x, float %y, float %z) { 692; SSE-LABEL: div_sqrt_fabs_f32: 693; SSE: # %bb.0: 694; SSE-NEXT: mulss %xmm1, %xmm1 695; SSE-NEXT: mulss %xmm2, %xmm1 696; SSE-NEXT: xorps %xmm2, %xmm2 697; SSE-NEXT: rsqrtss %xmm1, %xmm2 698; SSE-NEXT: mulss %xmm2, %xmm1 699; SSE-NEXT: mulss %xmm2, %xmm1 700; SSE-NEXT: addss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 701; SSE-NEXT: mulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 702; SSE-NEXT: mulss %xmm2, %xmm0 703; SSE-NEXT: mulss %xmm1, %xmm0 704; SSE-NEXT: retq 705; 706; AVX1-LABEL: div_sqrt_fabs_f32: 707; AVX1: # %bb.0: 708; AVX1-NEXT: vmulss %xmm1, %xmm1, %xmm1 709; AVX1-NEXT: vmulss %xmm2, %xmm1, %xmm1 710; AVX1-NEXT: vrsqrtss %xmm1, %xmm1, %xmm2 711; AVX1-NEXT: vmulss %xmm2, %xmm1, %xmm1 712; AVX1-NEXT: vmulss %xmm2, %xmm1, %xmm1 713; AVX1-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 714; AVX1-NEXT: vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 715; AVX1-NEXT: vmulss %xmm0, %xmm2, %xmm0 716; AVX1-NEXT: vmulss %xmm0, %xmm1, %xmm0 717; AVX1-NEXT: retq 718; 719; AVX512-LABEL: div_sqrt_fabs_f32: 720; AVX512: # %bb.0: 721; AVX512-NEXT: vmulss %xmm1, %xmm1, %xmm1 722; AVX512-NEXT: vmulss %xmm2, %xmm1, %xmm1 723; AVX512-NEXT: vrsqrtss %xmm1, %xmm1, %xmm2 724; AVX512-NEXT: vmulss %xmm2, %xmm1, %xmm1 725; AVX512-NEXT: vfmadd213ss {{.*#+}} xmm1 = (xmm2 * xmm1) + mem 726; AVX512-NEXT: vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 727; AVX512-NEXT: vmulss %xmm0, %xmm2, %xmm0 728; AVX512-NEXT: vmulss %xmm0, %xmm1, %xmm0 729; AVX512-NEXT: retq 730 %s = call fast float @llvm.sqrt.f32(float %z) 731 %a = call fast float @llvm.fabs.f32(float %y) 732 %m = fmul fast float %s, %a 733 %d = fdiv fast float %x, %m 734 ret float %d 735} 736 737; x / (fabs(y) * sqrt(z)) --> x * rsqrt(y*y*z) 738 739define <4 x float> @div_sqrt_fabs_v4f32(<4 x float> %x, <4 x float> %y, <4 x float> %z) { 740; SSE-LABEL: div_sqrt_fabs_v4f32: 741; SSE: # %bb.0: 742; SSE-NEXT: mulps %xmm1, %xmm1 743; SSE-NEXT: mulps %xmm2, %xmm1 744; SSE-NEXT: rsqrtps %xmm1, %xmm2 745; SSE-NEXT: mulps %xmm2, %xmm1 746; SSE-NEXT: mulps %xmm2, %xmm1 747; SSE-NEXT: addps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 748; SSE-NEXT: mulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 749; SSE-NEXT: mulps %xmm1, %xmm2 750; SSE-NEXT: mulps %xmm2, %xmm0 751; SSE-NEXT: retq 752; 753; AVX1-LABEL: div_sqrt_fabs_v4f32: 754; AVX1: # %bb.0: 755; AVX1-NEXT: vmulps %xmm1, %xmm1, %xmm1 756; AVX1-NEXT: vmulps %xmm2, %xmm1, %xmm1 757; AVX1-NEXT: vrsqrtps %xmm1, %xmm2 758; AVX1-NEXT: vmulps %xmm2, %xmm1, %xmm1 759; AVX1-NEXT: vmulps %xmm2, %xmm1, %xmm1 760; AVX1-NEXT: vaddps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 761; AVX1-NEXT: vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 762; AVX1-NEXT: vmulps %xmm1, %xmm2, %xmm1 763; AVX1-NEXT: vmulps %xmm1, %xmm0, %xmm0 764; AVX1-NEXT: retq 765; 766; AVX512-LABEL: div_sqrt_fabs_v4f32: 767; AVX512: # %bb.0: 768; AVX512-NEXT: vmulps %xmm1, %xmm1, %xmm1 769; AVX512-NEXT: vmulps %xmm2, %xmm1, %xmm1 770; AVX512-NEXT: vrsqrtps %xmm1, %xmm2 771; AVX512-NEXT: vmulps %xmm2, %xmm1, %xmm1 772; AVX512-NEXT: vbroadcastss {{.*#+}} xmm3 = [-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0] 773; AVX512-NEXT: vfmadd231ps {{.*#+}} xmm3 = (xmm2 * xmm1) + xmm3 774; AVX512-NEXT: vbroadcastss {{.*#+}} xmm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1] 775; AVX512-NEXT: vmulps %xmm1, %xmm2, %xmm1 776; AVX512-NEXT: vmulps %xmm3, %xmm1, %xmm1 777; AVX512-NEXT: vmulps %xmm1, %xmm0, %xmm0 778; AVX512-NEXT: retq 779 %s = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %z) 780 %a = call <4 x float> @llvm.fabs.v4f32(<4 x float> %y) 781 %m = fmul contract reassoc <4 x float> %a, %s 782 %d = fdiv contract reassoc arcp <4 x float> %x, %m 783 ret <4 x float> %d 784} 785 786; This has 'arcp' but does not have 'reassoc' FMF. 787; We allow converting the sqrt to an estimate, but 788; do not pull the divisor into the estimate. 789; x / (fabs(y) * sqrt(z)) --> x * rsqrt(z) / fabs(y) 790 791define <4 x float> @div_sqrt_fabs_v4f32_fmf(<4 x float> %x, <4 x float> %y, <4 x float> %z) { 792; SSE-LABEL: div_sqrt_fabs_v4f32_fmf: 793; SSE: # %bb.0: 794; SSE-NEXT: rsqrtps %xmm2, %xmm3 795; SSE-NEXT: mulps %xmm3, %xmm2 796; SSE-NEXT: mulps %xmm3, %xmm2 797; SSE-NEXT: addps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 798; SSE-NEXT: mulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 799; SSE-NEXT: andps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 800; SSE-NEXT: mulps %xmm2, %xmm3 801; SSE-NEXT: divps %xmm1, %xmm3 802; SSE-NEXT: mulps %xmm3, %xmm0 803; SSE-NEXT: retq 804; 805; AVX1-LABEL: div_sqrt_fabs_v4f32_fmf: 806; AVX1: # %bb.0: 807; AVX1-NEXT: vrsqrtps %xmm2, %xmm3 808; AVX1-NEXT: vmulps %xmm3, %xmm2, %xmm2 809; AVX1-NEXT: vmulps %xmm3, %xmm2, %xmm2 810; AVX1-NEXT: vaddps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 811; AVX1-NEXT: vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3, %xmm3 812; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 813; AVX1-NEXT: vmulps %xmm2, %xmm3, %xmm2 814; AVX1-NEXT: vdivps %xmm1, %xmm2, %xmm1 815; AVX1-NEXT: vmulps %xmm1, %xmm0, %xmm0 816; AVX1-NEXT: retq 817; 818; AVX512-LABEL: div_sqrt_fabs_v4f32_fmf: 819; AVX512: # %bb.0: 820; AVX512-NEXT: vrsqrtps %xmm2, %xmm3 821; AVX512-NEXT: vbroadcastss {{.*#+}} xmm4 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1] 822; AVX512-NEXT: vmulps %xmm4, %xmm3, %xmm4 823; AVX512-NEXT: vmulps %xmm3, %xmm2, %xmm2 824; AVX512-NEXT: vmulps %xmm3, %xmm2, %xmm2 825; AVX512-NEXT: vbroadcastss {{.*#+}} xmm3 = [-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0] 826; AVX512-NEXT: vaddps %xmm3, %xmm2, %xmm2 827; AVX512-NEXT: vmulps %xmm2, %xmm4, %xmm2 828; AVX512-NEXT: vbroadcastss {{.*#+}} xmm3 = [NaN,NaN,NaN,NaN] 829; AVX512-NEXT: vandps %xmm3, %xmm1, %xmm1 830; AVX512-NEXT: vdivps %xmm1, %xmm2, %xmm1 831; AVX512-NEXT: vmulps %xmm1, %xmm0, %xmm0 832; AVX512-NEXT: retq 833 %s = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %z) 834 %a = call <4 x float> @llvm.fabs.v4f32(<4 x float> %y) 835 %m = fmul <4 x float> %a, %s 836 %d = fdiv arcp <4 x float> %x, %m 837 ret <4 x float> %d 838} 839 840; No estimates for f64, so do not convert fabs into an fmul. 841 842define double @div_sqrt_fabs_f64(double %x, double %y, double %z) { 843; SSE-LABEL: div_sqrt_fabs_f64: 844; SSE: # %bb.0: 845; SSE-NEXT: andpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 846; SSE-NEXT: sqrtsd %xmm2, %xmm2 847; SSE-NEXT: mulsd %xmm2, %xmm1 848; SSE-NEXT: divsd %xmm1, %xmm0 849; SSE-NEXT: retq 850; 851; AVX-LABEL: div_sqrt_fabs_f64: 852; AVX: # %bb.0: 853; AVX-NEXT: vandpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 854; AVX-NEXT: vsqrtsd %xmm2, %xmm2, %xmm2 855; AVX-NEXT: vmulsd %xmm1, %xmm2, %xmm1 856; AVX-NEXT: vdivsd %xmm1, %xmm0, %xmm0 857; AVX-NEXT: retq 858 %s = call fast double @llvm.sqrt.f64(double %z) 859 %a = call fast double @llvm.fabs.f64(double %y) 860 %m = fmul fast double %s, %a 861 %d = fdiv fast double %x, %m 862 ret double %d 863} 864 865; This is a special case for the general pattern above - 866; if the sqrt operand is the same as the other mul op, 867; then fabs may be omitted. 868; x / (y * sqrt(y)) --> x * rsqrt(y*y*y) 869 870define float @div_sqrt_f32(float %x, float %y) { 871; SSE-LABEL: div_sqrt_f32: 872; SSE: # %bb.0: 873; SSE-NEXT: movaps %xmm1, %xmm2 874; SSE-NEXT: mulss %xmm1, %xmm2 875; SSE-NEXT: mulss %xmm1, %xmm2 876; SSE-NEXT: xorps %xmm1, %xmm1 877; SSE-NEXT: rsqrtss %xmm2, %xmm1 878; SSE-NEXT: mulss %xmm1, %xmm2 879; SSE-NEXT: mulss %xmm1, %xmm2 880; SSE-NEXT: addss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 881; SSE-NEXT: mulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 882; SSE-NEXT: mulss %xmm1, %xmm0 883; SSE-NEXT: mulss %xmm2, %xmm0 884; SSE-NEXT: retq 885; 886; AVX1-LABEL: div_sqrt_f32: 887; AVX1: # %bb.0: 888; AVX1-NEXT: vmulss %xmm1, %xmm1, %xmm2 889; AVX1-NEXT: vmulss %xmm1, %xmm2, %xmm1 890; AVX1-NEXT: vrsqrtss %xmm1, %xmm1, %xmm2 891; AVX1-NEXT: vmulss %xmm2, %xmm1, %xmm1 892; AVX1-NEXT: vmulss %xmm2, %xmm1, %xmm1 893; AVX1-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 894; AVX1-NEXT: vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 895; AVX1-NEXT: vmulss %xmm0, %xmm2, %xmm0 896; AVX1-NEXT: vmulss %xmm0, %xmm1, %xmm0 897; AVX1-NEXT: retq 898; 899; AVX512-LABEL: div_sqrt_f32: 900; AVX512: # %bb.0: 901; AVX512-NEXT: vmulss %xmm1, %xmm1, %xmm2 902; AVX512-NEXT: vmulss %xmm1, %xmm2, %xmm1 903; AVX512-NEXT: vrsqrtss %xmm1, %xmm1, %xmm2 904; AVX512-NEXT: vmulss %xmm2, %xmm1, %xmm1 905; AVX512-NEXT: vfmadd213ss {{.*#+}} xmm1 = (xmm2 * xmm1) + mem 906; AVX512-NEXT: vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 907; AVX512-NEXT: vmulss %xmm0, %xmm2, %xmm0 908; AVX512-NEXT: vmulss %xmm0, %xmm1, %xmm0 909; AVX512-NEXT: retq 910 %s = call fast float @llvm.sqrt.f32(float %y) 911 %m = fmul fast float %s, %y 912 %d = fdiv fast float %x, %m 913 ret float %d 914} 915 916; This is a special case for the general pattern above - 917; if the sqrt operand is the same as the other mul op, 918; then fabs may be omitted. 919; x / (y * sqrt(y)) --> x * rsqrt(y*y*y) 920 921define <4 x float> @div_sqrt_v4f32(<4 x float> %x, <4 x float> %y) { 922; SSE-LABEL: div_sqrt_v4f32: 923; SSE: # %bb.0: 924; SSE-NEXT: movaps %xmm1, %xmm2 925; SSE-NEXT: mulps %xmm1, %xmm2 926; SSE-NEXT: mulps %xmm1, %xmm2 927; SSE-NEXT: rsqrtps %xmm2, %xmm1 928; SSE-NEXT: mulps %xmm1, %xmm2 929; SSE-NEXT: mulps %xmm1, %xmm2 930; SSE-NEXT: addps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 931; SSE-NEXT: mulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 932; SSE-NEXT: mulps %xmm2, %xmm1 933; SSE-NEXT: mulps %xmm1, %xmm0 934; SSE-NEXT: retq 935; 936; AVX1-LABEL: div_sqrt_v4f32: 937; AVX1: # %bb.0: 938; AVX1-NEXT: vmulps %xmm1, %xmm1, %xmm2 939; AVX1-NEXT: vmulps %xmm1, %xmm2, %xmm1 940; AVX1-NEXT: vrsqrtps %xmm1, %xmm2 941; AVX1-NEXT: vmulps %xmm2, %xmm1, %xmm1 942; AVX1-NEXT: vmulps %xmm2, %xmm1, %xmm1 943; AVX1-NEXT: vaddps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 944; AVX1-NEXT: vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 945; AVX1-NEXT: vmulps %xmm1, %xmm2, %xmm1 946; AVX1-NEXT: vmulps %xmm1, %xmm0, %xmm0 947; AVX1-NEXT: retq 948; 949; AVX512-LABEL: div_sqrt_v4f32: 950; AVX512: # %bb.0: 951; AVX512-NEXT: vmulps %xmm1, %xmm1, %xmm2 952; AVX512-NEXT: vmulps %xmm1, %xmm2, %xmm1 953; AVX512-NEXT: vrsqrtps %xmm1, %xmm2 954; AVX512-NEXT: vmulps %xmm2, %xmm1, %xmm1 955; AVX512-NEXT: vbroadcastss {{.*#+}} xmm3 = [-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0] 956; AVX512-NEXT: vfmadd231ps {{.*#+}} xmm3 = (xmm2 * xmm1) + xmm3 957; AVX512-NEXT: vbroadcastss {{.*#+}} xmm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1] 958; AVX512-NEXT: vmulps %xmm1, %xmm2, %xmm1 959; AVX512-NEXT: vmulps %xmm3, %xmm1, %xmm1 960; AVX512-NEXT: vmulps %xmm1, %xmm0, %xmm0 961; AVX512-NEXT: retq 962 %s = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %y) 963 %m = fmul contract reassoc <4 x float> %y, %s 964 %d = fdiv contract reassoc arcp <4 x float> %x, %m 965 ret <4 x float> %d 966} 967 968define double @sqrt_fdiv_common_operand(double %x) nounwind { 969; SSE-LABEL: sqrt_fdiv_common_operand: 970; SSE: # %bb.0: 971; SSE-NEXT: sqrtsd %xmm0, %xmm0 972; SSE-NEXT: retq 973; 974; AVX-LABEL: sqrt_fdiv_common_operand: 975; AVX: # %bb.0: 976; AVX-NEXT: vsqrtsd %xmm0, %xmm0, %xmm0 977; AVX-NEXT: retq 978 %sqrt = call fast double @llvm.sqrt.f64(double %x) 979 %r = fdiv fast double %x, %sqrt 980 ret double %r 981} 982 983define <2 x double> @sqrt_fdiv_common_operand_vec(<2 x double> %x) nounwind { 984; SSE-LABEL: sqrt_fdiv_common_operand_vec: 985; SSE: # %bb.0: 986; SSE-NEXT: sqrtpd %xmm0, %xmm0 987; SSE-NEXT: retq 988; 989; AVX-LABEL: sqrt_fdiv_common_operand_vec: 990; AVX: # %bb.0: 991; AVX-NEXT: vsqrtpd %xmm0, %xmm0 992; AVX-NEXT: retq 993 %sqrt = call <2 x double> @llvm.sqrt.v2f64(<2 x double> %x) 994 %r = fdiv arcp nsz reassoc <2 x double> %x, %sqrt 995 ret <2 x double> %r 996} 997 998define double @sqrt_fdiv_common_operand_extra_use(double %x, ptr %p) nounwind { 999; SSE-LABEL: sqrt_fdiv_common_operand_extra_use: 1000; SSE: # %bb.0: 1001; SSE-NEXT: sqrtsd %xmm0, %xmm0 1002; SSE-NEXT: movsd %xmm0, (%rdi) 1003; SSE-NEXT: retq 1004; 1005; AVX-LABEL: sqrt_fdiv_common_operand_extra_use: 1006; AVX: # %bb.0: 1007; AVX-NEXT: vsqrtsd %xmm0, %xmm0, %xmm0 1008; AVX-NEXT: vmovsd %xmm0, (%rdi) 1009; AVX-NEXT: retq 1010 %sqrt = call fast double @llvm.sqrt.f64(double %x) 1011 store double %sqrt, ptr %p 1012 %r = fdiv fast double %x, %sqrt 1013 ret double %r 1014} 1015 1016define double @sqrt_simplify_before_recip(double %x, ptr %p) nounwind { 1017; SSE-LABEL: sqrt_simplify_before_recip: 1018; SSE: # %bb.0: 1019; SSE-NEXT: sqrtsd %xmm0, %xmm0 1020; SSE-NEXT: movsd {{.*#+}} xmm1 = mem[0],zero 1021; SSE-NEXT: divsd %xmm0, %xmm1 1022; SSE-NEXT: movsd %xmm1, (%rdi) 1023; SSE-NEXT: retq 1024; 1025; AVX-LABEL: sqrt_simplify_before_recip: 1026; AVX: # %bb.0: 1027; AVX-NEXT: vsqrtsd %xmm0, %xmm0, %xmm0 1028; AVX-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero 1029; AVX-NEXT: vdivsd %xmm0, %xmm1, %xmm1 1030; AVX-NEXT: vmovsd %xmm1, (%rdi) 1031; AVX-NEXT: retq 1032 %sqrt = tail call fast double @llvm.sqrt.f64(double %x) 1033 %rsqrt = fdiv fast double 1.0, %sqrt 1034 %sqrt_fast = fdiv fast double %x, %sqrt 1035 store double %rsqrt, ptr %p, align 8 1036 ret double %sqrt_fast 1037} 1038 1039define <2 x double> @sqrt_simplify_before_recip_vec(<2 x double> %x, ptr %p) nounwind { 1040; SSE-LABEL: sqrt_simplify_before_recip_vec: 1041; SSE: # %bb.0: 1042; SSE-NEXT: sqrtpd %xmm0, %xmm0 1043; SSE-NEXT: movapd {{.*#+}} xmm1 = [1.0E+0,1.0E+0] 1044; SSE-NEXT: divpd %xmm0, %xmm1 1045; SSE-NEXT: movupd %xmm1, (%rdi) 1046; SSE-NEXT: retq 1047; 1048; AVX-LABEL: sqrt_simplify_before_recip_vec: 1049; AVX: # %bb.0: 1050; AVX-NEXT: vsqrtpd %xmm0, %xmm0 1051; AVX-NEXT: vmovapd {{.*#+}} xmm1 = [1.0E+0,1.0E+0] 1052; AVX-NEXT: vdivpd %xmm0, %xmm1, %xmm1 1053; AVX-NEXT: vmovupd %xmm1, (%rdi) 1054; AVX-NEXT: retq 1055 %sqrt = tail call fast <2 x double> @llvm.sqrt.v2f64(<2 x double> %x) 1056 %rsqrt = fdiv fast <2 x double> <double 1.0, double 1.0>, %sqrt 1057 %sqrt_fast = fdiv fast <2 x double> %x, %sqrt 1058 store <2 x double> %rsqrt, ptr %p, align 8 1059 ret <2 x double> %sqrt_fast 1060} 1061 1062define double @sqrt_simplify_before_recip_order(double %x, ptr %p) nounwind { 1063; SSE-LABEL: sqrt_simplify_before_recip_order: 1064; SSE: # %bb.0: 1065; SSE-NEXT: sqrtsd %xmm0, %xmm0 1066; SSE-NEXT: movsd {{.*#+}} xmm1 = mem[0],zero 1067; SSE-NEXT: divsd %xmm0, %xmm1 1068; SSE-NEXT: movsd %xmm1, (%rdi) 1069; SSE-NEXT: retq 1070; 1071; AVX-LABEL: sqrt_simplify_before_recip_order: 1072; AVX: # %bb.0: 1073; AVX-NEXT: vsqrtsd %xmm0, %xmm0, %xmm0 1074; AVX-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero 1075; AVX-NEXT: vdivsd %xmm0, %xmm1, %xmm1 1076; AVX-NEXT: vmovsd %xmm1, (%rdi) 1077; AVX-NEXT: retq 1078 %sqrt = tail call fast double @llvm.sqrt.f64(double %x) 1079 %sqrt_fast = fdiv fast double %x, %sqrt 1080 %rsqrt = fdiv fast double 42.0, %sqrt 1081 store double %rsqrt, ptr %p, align 8 1082 ret double %sqrt_fast 1083} 1084 1085attributes #0 = { "unsafe-fp-math"="true" "reciprocal-estimates"="!sqrtf,!vec-sqrtf,!divf,!vec-divf" } 1086attributes #1 = { "unsafe-fp-math"="true" "reciprocal-estimates"="sqrt,vec-sqrt" } 1087attributes #2 = { nounwind readnone } 1088attributes #3 = { "unsafe-fp-math"="true" "reciprocal-estimates"="sqrt,vec-sqrt" "denormal-fp-math"="ieee" } 1089attributes #4 = { "unsafe-fp-math"="true" "reciprocal-estimates"="sqrt,vec-sqrt" "denormal-fp-math"="ieee,preserve-sign" } 1090attributes #5 = { "unsafe-fp-math"="true" "reciprocal-estimates"="all:0" } 1091