1 //! A NaN-canonicalizing rewriting pass. Patch floating point arithmetic
2 //! instructions that may return a NaN result with a sequence of operations
3 //! that will replace nondeterministic NaN's with a single canonical NaN value.
4 
5 use crate::cursor::{Cursor, FuncCursor};
6 use crate::ir::condcodes::FloatCC;
7 use crate::ir::immediates::{Ieee32, Ieee64};
8 use crate::ir::types::{self};
9 use crate::ir::{Function, Inst, InstBuilder, InstructionData, Opcode, Value};
10 use crate::opts::MemFlags;
11 use crate::timing;
12 
13 // Canonical 32-bit and 64-bit NaN values.
14 static CANON_32BIT_NAN: u32 = 0b01111111110000000000000000000000;
15 static CANON_64BIT_NAN: u64 = 0b0111111111111000000000000000000000000000000000000000000000000000;
16 
17 /// Perform the NaN canonicalization pass.
18 pub fn do_nan_canonicalization(func: &mut Function, has_vector_support: bool) {
19     let _tt = timing::canonicalize_nans();
20     let mut pos = FuncCursor::new(func);
21     while let Some(_block) = pos.next_block() {
22         while let Some(inst) = pos.next_inst() {
23             if is_fp_arith(&mut pos, inst) {
24                 add_nan_canon_seq(&mut pos, inst, has_vector_support);
25             }
26         }
27     }
28 }
29 
30 /// Returns true/false based on whether the instruction is a floating-point
31 /// arithmetic operation. This ignores operations like `fneg`, `fabs`, or
32 /// `fcopysign` that only operate on the sign bit of a floating point value.
33 fn is_fp_arith(pos: &mut FuncCursor, inst: Inst) -> bool {
34     match pos.func.dfg.insts[inst] {
35         InstructionData::Unary { opcode, .. } => {
36             opcode == Opcode::Ceil
37                 || opcode == Opcode::Floor
38                 || opcode == Opcode::Nearest
39                 || opcode == Opcode::Sqrt
40                 || opcode == Opcode::Trunc
41                 || opcode == Opcode::Fdemote
42                 || opcode == Opcode::Fpromote
43                 || opcode == Opcode::FvpromoteLow
44                 || opcode == Opcode::Fvdemote
45         }
46         InstructionData::Binary { opcode, .. } => {
47             opcode == Opcode::Fadd
48                 || opcode == Opcode::Fdiv
49                 || opcode == Opcode::Fmax
50                 || opcode == Opcode::Fmin
51                 || opcode == Opcode::Fmul
52                 || opcode == Opcode::Fsub
53         }
54         InstructionData::Ternary { opcode, .. } => opcode == Opcode::Fma,
55         _ => false,
56     }
57 }
58 
59 /// Append a sequence of canonicalizing instructions after the given instruction.
60 fn add_nan_canon_seq(pos: &mut FuncCursor, inst: Inst, has_vector_support: bool) {
61     // Select the instruction result, result type. Replace the instruction
62     // result and step forward before inserting the canonicalization sequence.
63     let val = pos.func.dfg.first_result(inst);
64     let val_type = pos.func.dfg.value_type(val);
65     let new_res = pos.func.dfg.replace_result(val, val_type);
66     let _next_inst = pos.next_inst().expect("block missing terminator!");
67 
68     // Insert a comparison instruction, to check if `inst_res` is NaN (comparing
69     // against NaN is always unordered). Select the canonical NaN value if `val`
70     // is NaN, assign the result to `inst`.
71     let comparison = FloatCC::Unordered;
72 
73     let vectorized_scalar_select = |pos: &mut FuncCursor, canon_nan: Value, ty: types::Type| {
74         let canon_nan = pos.ins().scalar_to_vector(ty, canon_nan);
75         let new_res = pos.ins().scalar_to_vector(ty, new_res);
76         let is_nan = pos.ins().fcmp(comparison, new_res, new_res);
77         let is_nan = pos.ins().bitcast(ty, MemFlags::new(), is_nan);
78         let simd_result = pos.ins().bitselect(is_nan, canon_nan, new_res);
79         pos.ins().with_result(val).extractlane(simd_result, 0);
80     };
81     let scalar_select = |pos: &mut FuncCursor, canon_nan: Value| {
82         let is_nan = pos.ins().fcmp(comparison, new_res, new_res);
83         pos.ins()
84             .with_result(val)
85             .select(is_nan, canon_nan, new_res);
86     };
87 
88     let vector_select = |pos: &mut FuncCursor, canon_nan: Value| {
89         let is_nan = pos.ins().fcmp(comparison, new_res, new_res);
90         let is_nan = pos.ins().bitcast(val_type, MemFlags::new(), is_nan);
91         pos.ins()
92             .with_result(val)
93             .bitselect(is_nan, canon_nan, new_res);
94     };
95 
96     match val_type {
97         types::F32 => {
98             let canon_nan = pos.ins().f32const(Ieee32::with_bits(CANON_32BIT_NAN));
99             if has_vector_support {
100                 vectorized_scalar_select(pos, canon_nan, types::F32X4);
101             } else {
102                 scalar_select(pos, canon_nan);
103             }
104         }
105         types::F64 => {
106             let canon_nan = pos.ins().f64const(Ieee64::with_bits(CANON_64BIT_NAN));
107             if has_vector_support {
108                 vectorized_scalar_select(pos, canon_nan, types::F64X2);
109             } else {
110                 scalar_select(pos, canon_nan);
111             }
112         }
113         types::F32X4 => {
114             let canon_nan = pos.ins().f32const(Ieee32::with_bits(CANON_32BIT_NAN));
115             let canon_nan = pos.ins().splat(types::F32X4, canon_nan);
116             vector_select(pos, canon_nan);
117         }
118         types::F64X2 => {
119             let canon_nan = pos.ins().f64const(Ieee64::with_bits(CANON_64BIT_NAN));
120             let canon_nan = pos.ins().splat(types::F64X2, canon_nan);
121             vector_select(pos, canon_nan);
122         }
123         _ => {
124             // Panic if the type given was not an IEEE floating point type.
125             panic!("Could not canonicalize NaN: Unexpected result type found.");
126         }
127     }
128 
129     pos.prev_inst(); // Step backwards so the pass does not skip instructions.
130 }
131