1; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=SI-UNSAFE -check-prefix=SI %s 2; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI-SAFE -check-prefix=SI %s 3 4declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone 5declare float @llvm.sqrt.f32(float) nounwind readnone 6declare double @llvm.sqrt.f64(double) nounwind readnone 7 8; SI-LABEL: {{^}}rsq_f32: 9; SI: v_rsq_f32_e32 10; SI: s_endpgm 11define amdgpu_kernel void @rsq_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #0 { 12 %val = load float, float addrspace(1)* %in, align 4 13 %sqrt = call float @llvm.sqrt.f32(float %val) nounwind readnone 14 %div = fdiv float 1.0, %sqrt, !fpmath !0 15 store float %div, float addrspace(1)* %out, align 4 16 ret void 17} 18 19; SI-LABEL: {{^}}rsq_f64: 20; SI: v_sqrt_f64 21; SI: v_rcp_f64 22; SI: s_endpgm 23define amdgpu_kernel void @rsq_f64(double addrspace(1)* noalias %out, double addrspace(1)* noalias %in) #0 { 24 %val = load double, double addrspace(1)* %in, align 4 25 %sqrt = call double @llvm.sqrt.f64(double %val) nounwind readnone 26 %div = fdiv double 1.0, %sqrt 27 store double %div, double addrspace(1)* %out, align 4 28 ret void 29} 30 31; SI-LABEL: {{^}}rsq_f32_sgpr: 32; SI: v_rsq_f32_e32 {{v[0-9]+}}, {{s[0-9]+}} 33; SI: s_endpgm 34define amdgpu_kernel void @rsq_f32_sgpr(float addrspace(1)* noalias %out, float %val) #0 { 35 %sqrt = call float @llvm.sqrt.f32(float %val) nounwind readnone 36 %div = fdiv float 1.0, %sqrt, !fpmath !0 37 store float %div, float addrspace(1)* %out, align 4 38 ret void 39} 40 41; Recognize that this is rsqrt(a) * rcp(b) * c, 42; not 1 / ( 1 / sqrt(a)) * rcp(b) * c. 43 44; NOTE: c * rcp( sqrt(a) * b ) is generated when we move rcp generation to AMGGPUCogenPrepare. 45 46; SI-LABEL: @rsqrt_fmul 47; SI-DAG: buffer_load_dword [[A:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}} 48; SI-DAG: buffer_load_dword [[B:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 49; SI-DAG: buffer_load_dword [[C:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 50 51; SI-UNSAFE-DAG: v_sqrt_f32_e32 [[SQRT:v[0-9]+]], [[A]] 52; SI-UNSAFE-DAG: v_mul_f32_e32 [[MUL:v[0-9]+]], [[SQRT]], [[B]] 53; SI-UNSAFE-DAG: v_rcp_f32_e32 [[RCP:v[0-9]+]], [[MUL]] 54; SI-UNSAFE-DAG: v_mul_f32_e32 [[RESULT:v[0-9]+]], [[C]], [[RCP]] 55; SI-UNSAFE: buffer_store_dword [[RESULT]] 56 57; SI-SAFE-NOT: v_rsq_f32 58 59; SI: s_endpgm 60define amdgpu_kernel void @rsqrt_fmul(float addrspace(1)* %out, float addrspace(1)* %in) #0 { 61 %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone 62 %out.gep = getelementptr float, float addrspace(1)* %out, i32 %tid 63 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid 64 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1 65 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2 66 67 %a = load volatile float, float addrspace(1)* %gep.0 68 %b = load volatile float, float addrspace(1)* %gep.1 69 %c = load volatile float, float addrspace(1)* %gep.2 70 71 %x = call float @llvm.sqrt.f32(float %a) 72 %y = fmul float %x, %b 73 %z = fdiv float %c, %y 74 store float %z, float addrspace(1)* %out.gep 75 ret void 76} 77 78; SI-LABEL: {{^}}neg_rsq_f32: 79; SI-SAFE: v_sqrt_f32_e32 [[SQRT:v[0-9]+]], v{{[0-9]+}} 80; SI-SAFE: v_rcp_f32_e64 [[RSQ:v[0-9]+]], -[[SQRT]] 81; SI-SAFE: buffer_store_dword [[RSQ]] 82 83; SI-UNSAFE: v_sqrt_f32_e32 [[SQRT:v[0-9]+]], v{{[0-9]+}} 84; SI-UNSAFE: v_rcp_f32_e64 [[RSQ:v[0-9]+]], -[[SQRT]] 85; SI-UNSAFE: buffer_store_dword [[RSQ]] 86define amdgpu_kernel void @neg_rsq_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #0 { 87 %val = load float, float addrspace(1)* %in, align 4 88 %sqrt = call float @llvm.sqrt.f32(float %val) 89 %div = fdiv float -1.0, %sqrt, !fpmath !0 90 store float %div, float addrspace(1)* %out, align 4 91 ret void 92} 93 94; SI-LABEL: {{^}}neg_rsq_f64: 95; SI-SAFE: v_sqrt_f64_e32 96; SI-SAFE: v_div_scale_f64 97 98; SI-UNSAFE: buffer_load_dwordx2 [[VAL:v\[[0-9]+:[0-9]+\]]] 99; SI-UNSAFE: v_sqrt_f64_e32 [[SQRT:v\[[0-9]+:[0-9]+\]]], [[VAL]] 100; SI-UNSAFE: v_rcp_f64_e32 [[RCP:v\[[0-9]+:[0-9]+\]]], [[VAL]] 101; SI-UNSAFE: v_fma_f64 {{v\[[0-9]+:[0-9]+\]}}, -{{v\[[0-9]+:[0-9]+\]}}, [[RCP]], 1.0 102; SI-UNSAFE: v_fma_f64 103; SI-UNSAFE: v_fma_f64 104; SI-UNSAFE: v_fma_f64 105; SI-UNSAFE: v_fma_f64 106; SI-UNSAFE: v_fma_f64 107define amdgpu_kernel void @neg_rsq_f64(double addrspace(1)* noalias %out, double addrspace(1)* noalias %in) #0 { 108 %val = load double, double addrspace(1)* %in, align 4 109 %sqrt = call double @llvm.sqrt.f64(double %val) 110 %div = fdiv double -1.0, %sqrt 111 store double %div, double addrspace(1)* %out, align 4 112 ret void 113} 114 115; SI-LABEL: {{^}}neg_rsq_neg_f32: 116; SI-SAFE: v_sqrt_f32_e64 [[SQRT:v[0-9]+]], -v{{[0-9]+}} 117; SI-SAFE: v_rcp_f32_e64 [[RSQ:v[0-9]+]], -[[SQRT]] 118; SI-SAFE: buffer_store_dword [[RSQ]] 119 120; SI-UNSAFE: v_sqrt_f32_e64 [[SQRT:v[0-9]+]], -v{{[0-9]+}} 121; SI-UNSAFE: v_rcp_f32_e64 [[RSQ:v[0-9]+]], -[[SQRT]] 122; SI-UNSAFE: buffer_store_dword [[RSQ]] 123define amdgpu_kernel void @neg_rsq_neg_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #0 { 124 %val = load float, float addrspace(1)* %in, align 4 125 %val.fneg = fneg float %val 126 %sqrt = call float @llvm.sqrt.f32(float %val.fneg) 127 %div = fdiv float -1.0, %sqrt, !fpmath !0 128 store float %div, float addrspace(1)* %out, align 4 129 ret void 130} 131 132; SI-LABEL: {{^}}neg_rsq_neg_f64: 133; SI-SAFE: v_sqrt_f64_e64 v{{\[[0-9]+:[0-9]+\]}}, -v{{\[[0-9]+:[0-9]+\]}} 134; SI-SAFE: v_div_scale_f64 135 136; SI-UNSAFE: buffer_load_dwordx2 [[VAL:v\[[0-9]+:[0-9]+\]]] 137; SI-UNSAFE-DAG: v_sqrt_f64_e64 [[SQRT:v\[[0-9]+:[0-9]+\]]], -[[VAL]] 138; SI-UNSAFE: v_rcp_f64_e32 [[RSQ:v\[[0-9]+:[0-9]+\]]], [[SQRT]] 139; SI-UNSAFE: v_fma_f64 {{v\[[0-9]+:[0-9]+\]}}, -{{v\[[0-9]+:[0-9]+\]}}, [[RSQ]], 1.0 140; SI-UNSAFE: v_fma_f64 141; SI-UNSAFE: v_fma_f64 142; SI-UNSAFE: v_fma_f64 143; SI-UNSAFE: v_fma_f64 144; SI-UNSAFE: v_fma_f64 145define amdgpu_kernel void @neg_rsq_neg_f64(double addrspace(1)* noalias %out, double addrspace(1)* noalias %in) #0 { 146 %val = load double, double addrspace(1)* %in, align 4 147 %val.fneg = fsub double -0.0, %val 148 %sqrt = call double @llvm.sqrt.f64(double %val.fneg) 149 %div = fdiv double -1.0, %sqrt 150 store double %div, double addrspace(1)* %out, align 4 151 ret void 152} 153 154!0 = !{float 2.500000e+00} 155 156attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" } 157