1 //! A NaN-canonicalizing rewriting pass. Patch floating point arithmetic
2 //! instructions that may return a NaN result with a sequence of operations
3 //! that will replace nondeterministic NaN's with a single canonical NaN value.
4 
5 use crate::cursor::{Cursor, FuncCursor};
6 use crate::ir::condcodes::FloatCC;
7 use crate::ir::immediates::{Ieee16, Ieee32, Ieee64, Ieee128};
8 use crate::ir::types::{self};
9 use crate::ir::{Function, Inst, InstBuilder, InstructionData, Opcode, Value};
10 use crate::opts::MemFlags;
11 use crate::timing;
12 
13 /// Perform the NaN canonicalization pass.
14 pub fn do_nan_canonicalization(func: &mut Function, has_vector_support: bool) {
15     let _tt = timing::canonicalize_nans();
16     let mut pos = FuncCursor::new(func);
17     while let Some(_block) = pos.next_block() {
18         while let Some(inst) = pos.next_inst() {
19             if is_fp_arith(&mut pos, inst) {
20                 add_nan_canon_seq(&mut pos, inst, has_vector_support);
21             }
22         }
23     }
24 }
25 
26 /// Returns true/false based on whether the instruction is a floating-point
27 /// arithmetic operation. This ignores operations like `fneg`, `fabs`, or
28 /// `fcopysign` that only operate on the sign bit of a floating point value.
29 fn is_fp_arith(pos: &mut FuncCursor, inst: Inst) -> bool {
30     match pos.func.dfg.insts[inst] {
31         InstructionData::Unary { opcode, .. } => {
32             opcode == Opcode::Ceil
33                 || opcode == Opcode::Floor
34                 || opcode == Opcode::Nearest
35                 || opcode == Opcode::Sqrt
36                 || opcode == Opcode::Trunc
37                 || opcode == Opcode::Fdemote
38                 || opcode == Opcode::Fpromote
39                 || opcode == Opcode::FvpromoteLow
40                 || opcode == Opcode::Fvdemote
41         }
42         InstructionData::Binary { opcode, .. } => {
43             opcode == Opcode::Fadd
44                 || opcode == Opcode::Fdiv
45                 || opcode == Opcode::Fmax
46                 || opcode == Opcode::Fmin
47                 || opcode == Opcode::Fmul
48                 || opcode == Opcode::Fsub
49         }
50         InstructionData::Ternary { opcode, .. } => opcode == Opcode::Fma,
51         _ => false,
52     }
53 }
54 
55 /// Append a sequence of canonicalizing instructions after the given instruction.
56 fn add_nan_canon_seq(pos: &mut FuncCursor, inst: Inst, has_vector_support: bool) {
57     // Select the instruction result, result type. Replace the instruction
58     // result and step forward before inserting the canonicalization sequence.
59     let val = pos.func.dfg.first_result(inst);
60     let val_type = pos.func.dfg.value_type(val);
61     let new_res = pos.func.dfg.replace_result(val, val_type);
62     let _next_inst = pos.next_inst().expect("block missing terminator!");
63 
64     // Insert a comparison instruction, to check if `inst_res` is NaN (comparing
65     // against NaN is always unordered). Select the canonical NaN value if `val`
66     // is NaN, assign the result to `inst`.
67     let comparison = FloatCC::Unordered;
68 
69     let vectorized_scalar_select = |pos: &mut FuncCursor, canon_nan: Value, ty: types::Type| {
70         let canon_nan = pos.ins().scalar_to_vector(ty, canon_nan);
71         let new_res = pos.ins().scalar_to_vector(ty, new_res);
72         let is_nan = pos.ins().fcmp(comparison, new_res, new_res);
73         let is_nan = pos.ins().bitcast(ty, MemFlags::new(), is_nan);
74         let simd_result = pos.ins().bitselect(is_nan, canon_nan, new_res);
75         pos.ins().with_result(val).extractlane(simd_result, 0);
76     };
77     let scalar_select = |pos: &mut FuncCursor, canon_nan: Value| {
78         let is_nan = pos.ins().fcmp(comparison, new_res, new_res);
79         pos.ins()
80             .with_result(val)
81             .select(is_nan, canon_nan, new_res);
82     };
83 
84     let vector_select = |pos: &mut FuncCursor, canon_nan: Value| {
85         let is_nan = pos.ins().fcmp(comparison, new_res, new_res);
86         let is_nan = pos.ins().bitcast(val_type, MemFlags::new(), is_nan);
87         pos.ins()
88             .with_result(val)
89             .bitselect(is_nan, canon_nan, new_res);
90     };
91 
92     match val_type {
93         types::F16 => {
94             let canon_nan = pos.ins().f16const(Ieee16::NAN);
95             scalar_select(pos, canon_nan);
96         }
97         types::F32 => {
98             let canon_nan = pos.ins().f32const(Ieee32::NAN);
99             if has_vector_support {
100                 vectorized_scalar_select(pos, canon_nan, types::F32X4);
101             } else {
102                 scalar_select(pos, canon_nan);
103             }
104         }
105         types::F64 => {
106             let canon_nan = pos.ins().f64const(Ieee64::NAN);
107             if has_vector_support {
108                 vectorized_scalar_select(pos, canon_nan, types::F64X2);
109             } else {
110                 scalar_select(pos, canon_nan);
111             }
112         }
113         types::F32X4 => {
114             let canon_nan = pos.ins().f32const(Ieee32::NAN);
115             let canon_nan = pos.ins().splat(types::F32X4, canon_nan);
116             vector_select(pos, canon_nan);
117         }
118         types::F64X2 => {
119             let canon_nan = pos.ins().f64const(Ieee64::NAN);
120             let canon_nan = pos.ins().splat(types::F64X2, canon_nan);
121             vector_select(pos, canon_nan);
122         }
123         types::F128 => {
124             let nan_const = pos.func.dfg.constants.insert(Ieee128::NAN.into());
125             let canon_nan = pos.ins().f128const(nan_const);
126             scalar_select(pos, canon_nan);
127         }
128         _ => {
129             // Panic if the type given was not an IEEE floating point type.
130             panic!("Could not canonicalize NaN: Unexpected result type found.");
131         }
132     }
133 
134     pos.prev_inst(); // Step backwards so the pass does not skip instructions.
135 }
136