1 //! The [DataValueExt] trait is an extension trait for [DataValue]. It provides a lot of functions
2 //! used by the rest of the interpreter.
3 
4 #![allow(trivial_numeric_casts)]
5 
6 use core::fmt::{self, Display, Formatter};
7 use core::ops::Neg;
8 use cranelift_codegen::data_value::{DataValue, DataValueCastFailure};
9 use cranelift_codegen::ir::immediates::{Ieee128, Ieee16, Ieee32, Ieee64};
10 use cranelift_codegen::ir::{types, Type};
11 use thiserror::Error;
12 
13 use crate::step::{extractlanes, SimdVec};
14 
15 pub type ValueResult<T> = Result<T, ValueError>;
16 
17 pub trait DataValueExt: Sized {
18     // Identity.
19     fn int(n: i128, ty: Type) -> ValueResult<Self>;
20     fn into_int_signed(self) -> ValueResult<i128>;
21     fn into_int_unsigned(self) -> ValueResult<u128>;
22     fn float(n: u64, ty: Type) -> ValueResult<Self>;
23     fn into_float(self) -> ValueResult<f64>;
24     fn is_float(&self) -> bool;
25     fn is_nan(&self) -> ValueResult<bool>;
26     fn bool(b: bool, vec_elem: bool, ty: Type) -> ValueResult<Self>;
27     fn into_bool(self) -> ValueResult<bool>;
28     fn vector(v: [u8; 16], ty: Type) -> ValueResult<Self>;
29     fn into_array(&self) -> ValueResult<[u8; 16]>;
30     fn convert(self, kind: ValueConversionKind) -> ValueResult<Self>;
31     fn concat(self, other: Self) -> ValueResult<Self>;
32 
33     fn is_negative(&self) -> ValueResult<bool>;
34     fn is_zero(&self) -> ValueResult<bool>;
35 
36     fn umax(self, other: Self) -> ValueResult<Self>;
37     fn smax(self, other: Self) -> ValueResult<Self>;
38     fn umin(self, other: Self) -> ValueResult<Self>;
39     fn smin(self, other: Self) -> ValueResult<Self>;
40 
41     // Comparison.
42     fn uno(&self, other: &Self) -> ValueResult<bool>;
43 
44     // Arithmetic.
45     fn add(self, other: Self) -> ValueResult<Self>;
46     fn sub(self, other: Self) -> ValueResult<Self>;
47     fn mul(self, other: Self) -> ValueResult<Self>;
48     fn udiv(self, other: Self) -> ValueResult<Self>;
49     fn sdiv(self, other: Self) -> ValueResult<Self>;
50     fn urem(self, other: Self) -> ValueResult<Self>;
51     fn srem(self, other: Self) -> ValueResult<Self>;
52     fn sqrt(self) -> ValueResult<Self>;
53     fn fma(self, a: Self, b: Self) -> ValueResult<Self>;
54     fn abs(self) -> ValueResult<Self>;
55     fn uadd_checked(self, other: Self) -> ValueResult<Option<Self>>;
56     fn sadd_checked(self, other: Self) -> ValueResult<Option<Self>>;
57     fn uadd_overflow(self, other: Self) -> ValueResult<(Self, bool)>;
58     fn sadd_overflow(self, other: Self) -> ValueResult<(Self, bool)>;
59     fn usub_overflow(self, other: Self) -> ValueResult<(Self, bool)>;
60     fn ssub_overflow(self, other: Self) -> ValueResult<(Self, bool)>;
61     fn umul_overflow(self, other: Self) -> ValueResult<(Self, bool)>;
62     fn smul_overflow(self, other: Self) -> ValueResult<(Self, bool)>;
63 
64     // Float operations
65     fn neg(self) -> ValueResult<Self>;
66     fn copysign(self, sign: Self) -> ValueResult<Self>;
67     fn ceil(self) -> ValueResult<Self>;
68     fn floor(self) -> ValueResult<Self>;
69     fn trunc(self) -> ValueResult<Self>;
70     fn nearest(self) -> ValueResult<Self>;
71 
72     // Saturating arithmetic.
73     fn uadd_sat(self, other: Self) -> ValueResult<Self>;
74     fn sadd_sat(self, other: Self) -> ValueResult<Self>;
75     fn usub_sat(self, other: Self) -> ValueResult<Self>;
76     fn ssub_sat(self, other: Self) -> ValueResult<Self>;
77 
78     // Bitwise.
79     fn shl(self, other: Self) -> ValueResult<Self>;
80     fn ushr(self, other: Self) -> ValueResult<Self>;
81     fn sshr(self, other: Self) -> ValueResult<Self>;
82     fn rotl(self, other: Self) -> ValueResult<Self>;
83     fn rotr(self, other: Self) -> ValueResult<Self>;
84     fn and(self, other: Self) -> ValueResult<Self>;
85     fn or(self, other: Self) -> ValueResult<Self>;
86     fn xor(self, other: Self) -> ValueResult<Self>;
87     fn not(self) -> ValueResult<Self>;
88 
89     // Bit counting.
90     fn count_ones(self) -> ValueResult<Self>;
91     fn leading_ones(self) -> ValueResult<Self>;
92     fn leading_zeros(self) -> ValueResult<Self>;
93     fn trailing_zeros(self) -> ValueResult<Self>;
94     fn reverse_bits(self) -> ValueResult<Self>;
95     fn swap_bytes(self) -> ValueResult<Self>;
96 
97     // An iterator over the lanes of a SIMD type
98     fn iter_lanes(&self, ty: Type) -> ValueResult<DataValueIterator>;
99 }
100 
101 #[derive(Error, Debug, PartialEq)]
102 pub enum ValueError {
103     #[error("unable to convert type {1} into class {0}")]
104     InvalidType(ValueTypeClass, Type),
105     #[error("unable to convert value into type {0}")]
106     InvalidValue(Type),
107     #[error("unable to convert to primitive integer")]
108     InvalidInteger(#[from] std::num::TryFromIntError),
109     #[error("unable to cast data value")]
110     InvalidDataValueCast(#[from] DataValueCastFailure),
111     #[error("performed a division by zero")]
112     IntegerDivisionByZero,
113     #[error("performed a operation that overflowed this integer type")]
114     IntegerOverflow,
115 }
116 
117 #[derive(Debug, PartialEq)]
118 pub enum ValueTypeClass {
119     Integer,
120     Boolean,
121     Float,
122     Vector,
123 }
124 
125 impl Display for ValueTypeClass {
126     fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
127         match self {
128             ValueTypeClass::Integer => write!(f, "integer"),
129             ValueTypeClass::Boolean => write!(f, "boolean"),
130             ValueTypeClass::Float => write!(f, "float"),
131             ValueTypeClass::Vector => write!(f, "vector"),
132         }
133     }
134 }
135 
136 #[derive(Debug, Clone)]
137 pub enum ValueConversionKind {
138     /// Throw a [ValueError] if an exact conversion to [Type] is not possible; e.g. in `i32` to
139     /// `i16`, convert `0x00001234` to `0x1234`.
140     Exact(Type),
141     /// Truncate the value to fit into the specified [Type]; e.g. in `i16` to `i8`, `0x1234` becomes
142     /// `0x34`.
143     Truncate(Type),
144     ///  Similar to Truncate, but extracts from the top of the value; e.g. in a `i32` to `u8`,
145     /// `0x12345678` becomes `0x12`.
146     ExtractUpper(Type),
147     /// Convert to a larger integer type, extending the sign bit; e.g. in `i8` to `i16`, `0xff`
148     /// becomes `0xffff`.
149     SignExtend(Type),
150     /// Convert to a larger integer type, extending with zeroes; e.g. in `i8` to `i16`, `0xff`
151     /// becomes `0x00ff`.
152     ZeroExtend(Type),
153     /// Convert a floating point number by rounding to the nearest possible value with ties to even.
154     /// See `fdemote`, e.g.
155     RoundNearestEven(Type),
156     /// Converts an integer into a boolean, zero integers are converted into a
157     /// `false`, while other integers are converted into `true`. Booleans are passed through.
158     ToBoolean,
159     /// Converts an integer into either -1 or zero.
160     Mask(Type),
161 }
162 
163 /// Helper for creating match expressions over [DataValue].
164 macro_rules! unary_match {
165     ( $op:ident($arg1:expr); [ $( $data_value_ty:ident ),* ]; [ $( $return_value_ty:ident ),* ] ) => {
166         match $arg1 {
167             $( DataValue::$data_value_ty(a) => {
168                 Ok(DataValue::$data_value_ty($return_value_ty::try_from(a.$op()).unwrap()))
169             } )*
170             _ => unimplemented!()
171         }
172     };
173     ( $op:ident($arg1:expr); [ $( $data_value_ty:ident ),* ] ) => {
174         match $arg1 {
175             $( DataValue::$data_value_ty(a) => { Ok(DataValue::$data_value_ty(a.$op())) } )*
176             _ => unimplemented!()
177         }
178     };
179     ( $op:tt($arg1:expr); [ $( $data_value_ty:ident ),* ] ) => {
180         match $arg1 {
181             $( DataValue::$data_value_ty(a) => { Ok(DataValue::$data_value_ty($op a)) } )*
182             _ => unimplemented!()
183         }
184     };
185 }
186 macro_rules! binary_match {
187     ( $op:ident($arg1:expr, $arg2:expr); [ $( $data_value_ty:ident ),* ] ) => {
188         match ($arg1, $arg2) {
189             $( (DataValue::$data_value_ty(a), DataValue::$data_value_ty(b)) => { Ok(DataValue::$data_value_ty(a.$op(*b))) } )*
190             _ => unimplemented!()
191         }
192     };
193     ( $op:ident($arg1:expr, $arg2:expr); [ $( $data_value_ty:ident ),* ]; [ $( $op_type:ty ),* ] ) => {
194         match ($arg1, $arg2) {
195             $( (DataValue::$data_value_ty(a), DataValue::$data_value_ty(b)) => { Ok(DataValue::$data_value_ty((*a as $op_type).$op(*b as $op_type) as _)) } )*
196             _ => unimplemented!()
197         }
198     };
199     ( option $op:ident($arg1:expr, $arg2:expr); [ $( $data_value_ty:ident ),* ]; [ $( $op_type:ty ),* ] ) => {
200         match ($arg1, $arg2) {
201             $( (DataValue::$data_value_ty(a), DataValue::$data_value_ty(b)) => { Ok((*a as $op_type).$op(*b as $op_type).map(|v| DataValue::$data_value_ty(v as _))) } )*
202             _ => unimplemented!()
203         }
204     };
205     ( pair $op:ident($arg1:expr, $arg2:expr); [ $( $data_value_ty:ident ),* ]; [ $( $op_type:ty ),* ] ) => {
206         match ($arg1, $arg2) {
207             $( (DataValue::$data_value_ty(a), DataValue::$data_value_ty(b)) => {
208                 let (f, s) = (*a as $op_type).$op(*b as $op_type);
209                 Ok((DataValue::$data_value_ty(f as _), s))
210             } )*
211             _ => unimplemented!()
212         }
213     };
214     ( $op:tt($arg1:expr, $arg2:expr); [ $( $data_value_ty:ident ),* ] ) => {
215         match ($arg1, $arg2) {
216             $( (DataValue::$data_value_ty(a), DataValue::$data_value_ty(b)) => { Ok(DataValue::$data_value_ty(a $op b)) } )*
217             _ => unimplemented!()
218         }
219     };
220     ( $op:tt($arg1:expr, $arg2:expr); [ $( $data_value_ty:ident ),* ]; [ $( $op_type:ty ),* ] ) => {
221         match ($arg1, $arg2) {
222             $( (DataValue::$data_value_ty(a), DataValue::$data_value_ty(b)) => { Ok(DataValue::$data_value_ty(((*a as $op_type) $op (*b as $op_type)) as _)) } )*
223             _ => unimplemented!()
224         }
225     };
226     ( $op:tt($arg1:expr, $arg2:expr); [ $( $data_value_ty:ident ),* ]; [ $( $a_type:ty ),* ]; rhs: $rhs:tt,$rhs_type:ty ) => {
227         match ($arg1, $arg2) {
228             $( (DataValue::$data_value_ty(a), DataValue::$rhs(b)) => { Ok(DataValue::$data_value_ty((*a as $a_type).$op(*b as $rhs_type) as _)) } )*
229             _ => unimplemented!()
230         }
231     };
232     ( $op:ident($arg1:expr, $arg2:expr); unsigned integers ) => {
233         match ($arg1, $arg2) {
234             (DataValue::I8(a), DataValue::I8(b)) => { Ok(DataValue::I8((u8::try_from(*a)?.$op(u8::try_from(*b)?) as i8))) }
235             (DataValue::I16(a), DataValue::I16(b)) => { Ok(DataValue::I16((u16::try_from(*a)?.$op(u16::try_from(*b)?) as i16))) }
236             (DataValue::I32(a), DataValue::I32(b)) => { Ok(DataValue::I32((u32::try_from(*a)?.$op(u32::try_from(*b)?) as i32))) }
237             (DataValue::I64(a), DataValue::I64(b)) => { Ok(DataValue::I64((u64::try_from(*a)?.$op(u64::try_from(*b)?) as i64))) }
238             (DataValue::I128(a), DataValue::I128(b)) => { Ok(DataValue::I128((u128::try_from(*a)?.$op(u128::try_from(*b)?) as i64))) }
239             _ => { Err(ValueError::InvalidType(ValueTypeClass::Integer, if !($arg1).ty().is_int() { ($arg1).ty() } else { ($arg2).ty() })) }
240         }
241     };
242 }
243 
244 macro_rules! bitop {
245     ( $op:tt($arg1:expr, $arg2:expr) ) => {
246         Ok(match ($arg1, $arg2) {
247             (DataValue::I8(a), DataValue::I8(b)) => DataValue::I8(a $op b),
248             (DataValue::I16(a), DataValue::I16(b)) => DataValue::I16(a $op b),
249             (DataValue::I32(a), DataValue::I32(b)) => DataValue::I32(a $op b),
250             (DataValue::I64(a), DataValue::I64(b)) => DataValue::I64(a $op b),
251             (DataValue::I128(a), DataValue::I128(b)) => DataValue::I128(a $op b),
252             (DataValue::F32(a), DataValue::F32(b)) => DataValue::F32(a $op b),
253             (DataValue::F64(a), DataValue::F64(b)) => DataValue::F64(a $op b),
254             (DataValue::V128(a), DataValue::V128(b)) => {
255                 let mut a2 = a.clone();
256                 for (a, b) in a2.iter_mut().zip(b.iter()) {
257                     *a = *a $op *b;
258                 }
259                 DataValue::V128(a2)
260             }
261             _ => unimplemented!(),
262         })
263     };
264 }
265 
266 impl DataValueExt for DataValue {
267     fn int(n: i128, ty: Type) -> ValueResult<Self> {
268         if ty.is_vector() {
269             // match ensures graceful failure since read_from_slice_ne()
270             // panics on anything other than 8 and 16 bytes
271             match ty.bytes() {
272                 8 | 16 => Ok(DataValue::read_from_slice_ne(&n.to_ne_bytes(), ty)),
273                 _ => Err(ValueError::InvalidType(ValueTypeClass::Vector, ty)),
274             }
275         } else if ty.is_int() {
276             DataValue::from_integer(n, ty).map_err(|_| ValueError::InvalidValue(ty))
277         } else {
278             Err(ValueError::InvalidType(ValueTypeClass::Integer, ty))
279         }
280     }
281 
282     fn into_int_signed(self) -> ValueResult<i128> {
283         match self {
284             DataValue::I8(n) => Ok(n as i128),
285             DataValue::I16(n) => Ok(n as i128),
286             DataValue::I32(n) => Ok(n as i128),
287             DataValue::I64(n) => Ok(n as i128),
288             DataValue::I128(n) => Ok(n),
289             _ => Err(ValueError::InvalidType(ValueTypeClass::Integer, self.ty())),
290         }
291     }
292 
293     fn into_int_unsigned(self) -> ValueResult<u128> {
294         match self {
295             DataValue::I8(n) => Ok(n as u8 as u128),
296             DataValue::I16(n) => Ok(n as u16 as u128),
297             DataValue::I32(n) => Ok(n as u32 as u128),
298             DataValue::I64(n) => Ok(n as u64 as u128),
299             DataValue::I128(n) => Ok(n as u128),
300             _ => Err(ValueError::InvalidType(ValueTypeClass::Integer, self.ty())),
301         }
302     }
303 
304     fn float(bits: u64, ty: Type) -> ValueResult<Self> {
305         match ty {
306             types::F32 => Ok(DataValue::F32(Ieee32::with_bits(u32::try_from(bits)?))),
307             types::F64 => Ok(DataValue::F64(Ieee64::with_bits(bits))),
308             _ => Err(ValueError::InvalidType(ValueTypeClass::Float, ty)),
309         }
310     }
311 
312     fn into_float(self) -> ValueResult<f64> {
313         match self {
314             DataValue::F32(n) => Ok(n.as_f32() as f64),
315             DataValue::F64(n) => Ok(n.as_f64()),
316             _ => Err(ValueError::InvalidType(ValueTypeClass::Float, self.ty())),
317         }
318     }
319 
320     fn is_float(&self) -> bool {
321         match self {
322             DataValue::F16(_) | DataValue::F32(_) | DataValue::F64(_) | DataValue::F128(_) => true,
323             _ => false,
324         }
325     }
326 
327     fn is_nan(&self) -> ValueResult<bool> {
328         match self {
329             DataValue::F32(f) => Ok(f.is_nan()),
330             DataValue::F64(f) => Ok(f.is_nan()),
331             _ => Err(ValueError::InvalidType(ValueTypeClass::Float, self.ty())),
332         }
333     }
334 
335     fn bool(b: bool, vec_elem: bool, ty: Type) -> ValueResult<Self> {
336         assert!(ty.is_int());
337         macro_rules! make_bool {
338             ($ty:ident) => {
339                 Ok(DataValue::$ty(if b {
340                     if vec_elem {
341                         -1
342                     } else {
343                         1
344                     }
345                 } else {
346                     0
347                 }))
348             };
349         }
350 
351         match ty {
352             types::I8 => make_bool!(I8),
353             types::I16 => make_bool!(I16),
354             types::I32 => make_bool!(I32),
355             types::I64 => make_bool!(I64),
356             types::I128 => make_bool!(I128),
357             _ => Err(ValueError::InvalidType(ValueTypeClass::Integer, ty)),
358         }
359     }
360 
361     fn into_bool(self) -> ValueResult<bool> {
362         match self {
363             DataValue::I8(b) => Ok(b != 0),
364             DataValue::I16(b) => Ok(b != 0),
365             DataValue::I32(b) => Ok(b != 0),
366             DataValue::I64(b) => Ok(b != 0),
367             DataValue::I128(b) => Ok(b != 0),
368             _ => Err(ValueError::InvalidType(ValueTypeClass::Boolean, self.ty())),
369         }
370     }
371 
372     fn vector(v: [u8; 16], ty: Type) -> ValueResult<Self> {
373         assert!(ty.is_vector() && [8, 16].contains(&ty.bytes()));
374         if ty.bytes() == 16 {
375             Ok(DataValue::V128(v))
376         } else if ty.bytes() == 8 {
377             let v64: [u8; 8] = v[..8].try_into().unwrap();
378             Ok(DataValue::V64(v64))
379         } else {
380             unimplemented!()
381         }
382     }
383 
384     fn into_array(&self) -> ValueResult<[u8; 16]> {
385         match *self {
386             DataValue::V128(v) => Ok(v),
387             DataValue::V64(v) => {
388                 let mut v128 = [0; 16];
389                 v128[..8].clone_from_slice(&v);
390                 Ok(v128)
391             }
392             _ => Err(ValueError::InvalidType(ValueTypeClass::Vector, self.ty())),
393         }
394     }
395 
396     fn convert(self, kind: ValueConversionKind) -> ValueResult<Self> {
397         Ok(match kind {
398             ValueConversionKind::Exact(ty) => match (self, ty) {
399                 // TODO a lot to do here: from bmask to ireduce to bitcast...
400                 (val, ty) if val.ty().is_int() && ty.is_int() => {
401                     DataValue::from_integer(val.into_int_signed()?, ty)?
402                 }
403                 (DataValue::I16(n), types::F16) => DataValue::F16(Ieee16::with_bits(n as u16)),
404                 (DataValue::I32(n), types::F32) => DataValue::F32(f32::from_bits(n as u32).into()),
405                 (DataValue::I64(n), types::F64) => DataValue::F64(f64::from_bits(n as u64).into()),
406                 (DataValue::I128(n), types::F128) => DataValue::F128(Ieee128::with_bits(n as u128)),
407                 (DataValue::F16(n), types::I16) => DataValue::I16(n.bits() as i16),
408                 (DataValue::F32(n), types::I32) => DataValue::I32(n.bits() as i32),
409                 (DataValue::F64(n), types::I64) => DataValue::I64(n.bits() as i64),
410                 (DataValue::F128(n), types::I128) => DataValue::I128(n.bits() as i128),
411                 (DataValue::F32(n), types::F64) => DataValue::F64((n.as_f32() as f64).into()),
412                 (dv, t) if (t.is_int() || t.is_float()) && dv.ty() == t => dv,
413                 (dv, _) => unimplemented!("conversion: {} -> {:?}", dv.ty(), kind),
414             },
415             ValueConversionKind::Truncate(ty) => {
416                 assert!(
417                     ty.is_int(),
418                     "unimplemented conversion: {} -> {:?}",
419                     self.ty(),
420                     kind
421                 );
422 
423                 let mask = (1 << (ty.bytes() * 8)) - 1i128;
424                 let truncated = self.into_int_signed()? & mask;
425                 Self::from_integer(truncated, ty)?
426             }
427             ValueConversionKind::ExtractUpper(ty) => {
428                 assert!(
429                     ty.is_int(),
430                     "unimplemented conversion: {} -> {:?}",
431                     self.ty(),
432                     kind
433                 );
434 
435                 let shift_amt = (self.ty().bytes() * 8) - (ty.bytes() * 8);
436                 let mask = (1 << (ty.bytes() * 8)) - 1i128;
437                 let shifted_mask = mask << shift_amt;
438 
439                 let extracted = (self.into_int_signed()? & shifted_mask) >> shift_amt;
440                 Self::from_integer(extracted, ty)?
441             }
442             ValueConversionKind::SignExtend(ty) => match (self, ty) {
443                 (DataValue::I8(n), types::I16) => DataValue::I16(n as i16),
444                 (DataValue::I8(n), types::I32) => DataValue::I32(n as i32),
445                 (DataValue::I8(n), types::I64) => DataValue::I64(n as i64),
446                 (DataValue::I8(n), types::I128) => DataValue::I128(n as i128),
447                 (DataValue::I16(n), types::I32) => DataValue::I32(n as i32),
448                 (DataValue::I16(n), types::I64) => DataValue::I64(n as i64),
449                 (DataValue::I16(n), types::I128) => DataValue::I128(n as i128),
450                 (DataValue::I32(n), types::I64) => DataValue::I64(n as i64),
451                 (DataValue::I32(n), types::I128) => DataValue::I128(n as i128),
452                 (DataValue::I64(n), types::I128) => DataValue::I128(n as i128),
453                 (dv, _) => unimplemented!("conversion: {} -> {:?}", dv.ty(), kind),
454             },
455             ValueConversionKind::ZeroExtend(ty) => match (self, ty) {
456                 (DataValue::I8(n), types::I16) => DataValue::I16(n as u8 as i16),
457                 (DataValue::I8(n), types::I32) => DataValue::I32(n as u8 as i32),
458                 (DataValue::I8(n), types::I64) => DataValue::I64(n as u8 as i64),
459                 (DataValue::I8(n), types::I128) => DataValue::I128(n as u8 as i128),
460                 (DataValue::I16(n), types::I32) => DataValue::I32(n as u16 as i32),
461                 (DataValue::I16(n), types::I64) => DataValue::I64(n as u16 as i64),
462                 (DataValue::I16(n), types::I128) => DataValue::I128(n as u16 as i128),
463                 (DataValue::I32(n), types::I64) => DataValue::I64(n as u32 as i64),
464                 (DataValue::I32(n), types::I128) => DataValue::I128(n as u32 as i128),
465                 (DataValue::I64(n), types::I128) => DataValue::I128(n as u64 as i128),
466                 (from, to) if from.ty() == to => from,
467                 (dv, _) => unimplemented!("conversion: {} -> {:?}", dv.ty(), kind),
468             },
469             ValueConversionKind::RoundNearestEven(ty) => match (self, ty) {
470                 (DataValue::F64(n), types::F32) => DataValue::F32(Ieee32::from(n.as_f64() as f32)),
471                 (s, _) => unimplemented!("conversion: {} -> {:?}", s.ty(), kind),
472             },
473             ValueConversionKind::ToBoolean => match self.ty() {
474                 ty if ty.is_int() => {
475                     DataValue::I8(if self.into_int_signed()? != 0 { 1 } else { 0 })
476                 }
477                 ty => unimplemented!("conversion: {} -> {:?}", ty, kind),
478             },
479             ValueConversionKind::Mask(ty) => {
480                 let b = self.into_bool()?;
481                 Self::bool(b, true, ty).unwrap()
482             }
483         })
484     }
485 
486     fn concat(self, other: Self) -> ValueResult<Self> {
487         match (self, other) {
488             (DataValue::I64(lhs), DataValue::I64(rhs)) => Ok(DataValue::I128(
489                 (((lhs as u64) as u128) | (((rhs as u64) as u128) << 64)) as i128,
490             )),
491             (lhs, rhs) => unimplemented!("concat: {} -> {}", lhs.ty(), rhs.ty()),
492         }
493     }
494 
495     fn is_negative(&self) -> ValueResult<bool> {
496         match self {
497             DataValue::F32(f) => Ok(f.is_negative()),
498             DataValue::F64(f) => Ok(f.is_negative()),
499             _ => Err(ValueError::InvalidType(ValueTypeClass::Float, self.ty())),
500         }
501     }
502 
503     fn is_zero(&self) -> ValueResult<bool> {
504         match self {
505             DataValue::I8(f) => Ok(*f == 0),
506             DataValue::I16(f) => Ok(*f == 0),
507             DataValue::I32(f) => Ok(*f == 0),
508             DataValue::I64(f) => Ok(*f == 0),
509             DataValue::I128(f) => Ok(*f == 0),
510             DataValue::F16(f) => Ok(f.is_zero()),
511             DataValue::F32(f) => Ok(f.is_zero()),
512             DataValue::F64(f) => Ok(f.is_zero()),
513             DataValue::F128(f) => Ok(f.is_zero()),
514             DataValue::V64(_) | DataValue::V128(_) => {
515                 Err(ValueError::InvalidType(ValueTypeClass::Float, self.ty()))
516             }
517         }
518     }
519 
520     fn umax(self, other: Self) -> ValueResult<Self> {
521         let lhs = self.clone().into_int_unsigned()?;
522         let rhs = other.clone().into_int_unsigned()?;
523         if lhs > rhs {
524             Ok(self)
525         } else {
526             Ok(other)
527         }
528     }
529 
530     fn smax(self, other: Self) -> ValueResult<Self> {
531         if self > other {
532             Ok(self)
533         } else {
534             Ok(other)
535         }
536     }
537 
538     fn umin(self, other: Self) -> ValueResult<Self> {
539         let lhs = self.clone().into_int_unsigned()?;
540         let rhs = other.clone().into_int_unsigned()?;
541         if lhs < rhs {
542             Ok(self)
543         } else {
544             Ok(other)
545         }
546     }
547 
548     fn smin(self, other: Self) -> ValueResult<Self> {
549         if self < other {
550             Ok(self)
551         } else {
552             Ok(other)
553         }
554     }
555 
556     fn uno(&self, other: &Self) -> ValueResult<bool> {
557         Ok(self.is_nan()? || other.is_nan()?)
558     }
559 
560     fn add(self, other: Self) -> ValueResult<Self> {
561         if self.is_float() {
562             binary_match!(+(self, other); [F32, F64])
563         } else {
564             binary_match!(wrapping_add(&self, &other); [I8, I16, I32, I64, I128])
565         }
566     }
567 
568     fn sub(self, other: Self) -> ValueResult<Self> {
569         if self.is_float() {
570             binary_match!(-(self, other); [F32, F64])
571         } else {
572             binary_match!(wrapping_sub(&self, &other); [I8, I16, I32, I64, I128])
573         }
574     }
575 
576     fn mul(self, other: Self) -> ValueResult<Self> {
577         if self.is_float() {
578             binary_match!(*(self, other); [F32, F64])
579         } else {
580             binary_match!(wrapping_mul(&self, &other); [I8, I16, I32, I64, I128])
581         }
582     }
583 
584     fn sdiv(self, other: Self) -> ValueResult<Self> {
585         if self.is_float() {
586             return binary_match!(/(self, other); [F32, F64]);
587         }
588 
589         let denominator = other.clone().into_int_signed()?;
590 
591         // Check if we are dividing INT_MIN / -1. This causes an integer overflow trap.
592         let min = DataValueExt::int(1i128 << (self.ty().bits() - 1), self.ty())?;
593         if self == min && denominator == -1 {
594             return Err(ValueError::IntegerOverflow);
595         }
596 
597         if denominator == 0 {
598             return Err(ValueError::IntegerDivisionByZero);
599         }
600 
601         binary_match!(/(&self, &other); [I8, I16, I32, I64, I128])
602     }
603 
604     fn udiv(self, other: Self) -> ValueResult<Self> {
605         if self.is_float() {
606             return binary_match!(/(self, other); [F32, F64]);
607         }
608 
609         let denominator = other.clone().into_int_unsigned()?;
610 
611         if denominator == 0 {
612             return Err(ValueError::IntegerDivisionByZero);
613         }
614 
615         binary_match!(/(&self, &other); [I8, I16, I32, I64, I128]; [u8, u16, u32, u64, u128])
616     }
617 
618     fn srem(self, other: Self) -> ValueResult<Self> {
619         let denominator = other.clone().into_int_signed()?;
620 
621         // Check if we are dividing INT_MIN / -1. This causes an integer overflow trap.
622         let min = DataValueExt::int(1i128 << (self.ty().bits() - 1), self.ty())?;
623         if self == min && denominator == -1 {
624             return Err(ValueError::IntegerOverflow);
625         }
626 
627         if denominator == 0 {
628             return Err(ValueError::IntegerDivisionByZero);
629         }
630 
631         binary_match!(%(&self, &other); [I8, I16, I32, I64, I128])
632     }
633 
634     fn urem(self, other: Self) -> ValueResult<Self> {
635         let denominator = other.clone().into_int_unsigned()?;
636 
637         if denominator == 0 {
638             return Err(ValueError::IntegerDivisionByZero);
639         }
640 
641         binary_match!(%(&self, &other); [I8, I16, I32, I64, I128]; [u8, u16, u32, u64, u128])
642     }
643 
644     fn sqrt(self) -> ValueResult<Self> {
645         unary_match!(sqrt(&self); [F32, F64]; [Ieee32, Ieee64])
646     }
647 
648     fn fma(self, b: Self, c: Self) -> ValueResult<Self> {
649         match (self, b, c) {
650             (DataValue::F32(a), DataValue::F32(b), DataValue::F32(c)) => {
651                 // The `fma` function for `x86_64-pc-windows-gnu` is incorrect. Use `libm`'s instead.
652                 // See: https://github.com/bytecodealliance/wasmtime/issues/4512
653                 #[cfg(all(target_arch = "x86_64", target_os = "windows", target_env = "gnu"))]
654                 let res = libm::fmaf(a.as_f32(), b.as_f32(), c.as_f32());
655 
656                 #[cfg(not(all(
657                     target_arch = "x86_64",
658                     target_os = "windows",
659                     target_env = "gnu"
660                 )))]
661                 let res = a.as_f32().mul_add(b.as_f32(), c.as_f32());
662 
663                 Ok(DataValue::F32(res.into()))
664             }
665             (DataValue::F64(a), DataValue::F64(b), DataValue::F64(c)) => {
666                 #[cfg(all(target_arch = "x86_64", target_os = "windows", target_env = "gnu"))]
667                 let res = libm::fma(a.as_f64(), b.as_f64(), c.as_f64());
668 
669                 #[cfg(not(all(
670                     target_arch = "x86_64",
671                     target_os = "windows",
672                     target_env = "gnu"
673                 )))]
674                 let res = a.as_f64().mul_add(b.as_f64(), c.as_f64());
675 
676                 Ok(DataValue::F64(res.into()))
677             }
678             (a, _b, _c) => Err(ValueError::InvalidType(ValueTypeClass::Float, a.ty())),
679         }
680     }
681 
682     fn abs(self) -> ValueResult<Self> {
683         unary_match!(abs(&self); [F32, F64])
684     }
685 
686     fn sadd_checked(self, other: Self) -> ValueResult<Option<Self>> {
687         binary_match!(option checked_add(&self, &other); [I8, I16, I32, I64, I128]; [i8, i16, i32, i64, i128])
688     }
689 
690     fn uadd_checked(self, other: Self) -> ValueResult<Option<Self>> {
691         binary_match!(option checked_add(&self, &other); [I8, I16, I32, I64, I128]; [u8, u16, u32, u64, u128])
692     }
693 
694     fn sadd_overflow(self, other: Self) -> ValueResult<(Self, bool)> {
695         binary_match!(pair overflowing_add(&self, &other); [I8, I16, I32, I64, I128]; [i8, i16, i32, i64, i128])
696     }
697 
698     fn uadd_overflow(self, other: Self) -> ValueResult<(Self, bool)> {
699         binary_match!(pair overflowing_add(&self, &other); [I8, I16, I32, I64, I128]; [u8, u16, u32, u64, u128])
700     }
701 
702     fn ssub_overflow(self, other: Self) -> ValueResult<(Self, bool)> {
703         binary_match!(pair overflowing_sub(&self, &other); [I8, I16, I32, I64, I128]; [i8, i16, i32, i64, i128])
704     }
705 
706     fn usub_overflow(self, other: Self) -> ValueResult<(Self, bool)> {
707         binary_match!(pair overflowing_sub(&self, &other); [I8, I16, I32, I64, I128]; [u8, u16, u32, u64, u128])
708     }
709 
710     fn smul_overflow(self, other: Self) -> ValueResult<(Self, bool)> {
711         binary_match!(pair overflowing_mul(&self, &other); [I8, I16, I32, I64, I128]; [i8, i16, i32, i64, i128])
712     }
713 
714     fn umul_overflow(self, other: Self) -> ValueResult<(Self, bool)> {
715         binary_match!(pair overflowing_mul(&self, &other); [I8, I16, I32, I64, I128]; [u8, u16, u32, u64, u128])
716     }
717 
718     fn neg(self) -> ValueResult<Self> {
719         unary_match!(neg(&self); [F32, F64])
720     }
721 
722     fn copysign(self, sign: Self) -> ValueResult<Self> {
723         binary_match!(copysign(&self, &sign); [F32, F64])
724     }
725 
726     fn ceil(self) -> ValueResult<Self> {
727         unary_match!(ceil(&self); [F32, F64])
728     }
729 
730     fn floor(self) -> ValueResult<Self> {
731         unary_match!(floor(&self); [F32, F64])
732     }
733 
734     fn trunc(self) -> ValueResult<Self> {
735         unary_match!(trunc(&self); [F32, F64])
736     }
737 
738     fn nearest(self) -> ValueResult<Self> {
739         unary_match!(round_ties_even(&self); [F32, F64])
740     }
741 
742     fn sadd_sat(self, other: Self) -> ValueResult<Self> {
743         binary_match!(saturating_add(self, &other); [I8, I16, I32, I64, I128])
744     }
745 
746     fn uadd_sat(self, other: Self) -> ValueResult<Self> {
747         binary_match!(saturating_add(&self, &other); [I8, I16, I32, I64, I128]; [u8, u16, u32, u64, u128])
748     }
749 
750     fn ssub_sat(self, other: Self) -> ValueResult<Self> {
751         binary_match!(saturating_sub(self, &other); [I8, I16, I32, I64, I128])
752     }
753 
754     fn usub_sat(self, other: Self) -> ValueResult<Self> {
755         binary_match!(saturating_sub(&self, &other); [I8, I16, I32, I64, I128]; [u8, u16, u32, u64, u128])
756     }
757 
758     fn shl(self, other: Self) -> ValueResult<Self> {
759         let amt = other.convert(ValueConversionKind::Exact(types::I32))?;
760         binary_match!(wrapping_shl(&self, &amt); [I8, I16, I32, I64, I128]; [i8, i16, i32, i64, i128]; rhs: I32,u32)
761     }
762 
763     fn ushr(self, other: Self) -> ValueResult<Self> {
764         let amt = other.convert(ValueConversionKind::Exact(types::I32))?;
765         binary_match!(wrapping_shr(&self, &amt); [I8, I16, I32, I64, I128]; [u8, u16, u32, u64, u128]; rhs: I32,u32)
766     }
767 
768     fn sshr(self, other: Self) -> ValueResult<Self> {
769         let amt = other.convert(ValueConversionKind::Exact(types::I32))?;
770         binary_match!(wrapping_shr(&self, &amt); [I8, I16, I32, I64, I128]; [i8, i16, i32, i64, i128]; rhs: I32,u32)
771     }
772 
773     fn rotl(self, other: Self) -> ValueResult<Self> {
774         let amt = other.convert(ValueConversionKind::Exact(types::I32))?;
775         binary_match!(rotate_left(&self, &amt); [I8, I16, I32, I64, I128]; [i8, i16, i32, i64, i128]; rhs: I32,u32)
776     }
777 
778     fn rotr(self, other: Self) -> ValueResult<Self> {
779         let amt = other.convert(ValueConversionKind::Exact(types::I32))?;
780         binary_match!(rotate_right(&self, &amt); [I8, I16, I32, I64, I128]; [i8, i16, i32, i64, i128]; rhs: I32,u32)
781     }
782 
783     fn and(self, other: Self) -> ValueResult<Self> {
784         bitop!(&(self, other))
785     }
786 
787     fn or(self, other: Self) -> ValueResult<Self> {
788         bitop!(|(self, other))
789     }
790 
791     fn xor(self, other: Self) -> ValueResult<Self> {
792         bitop!(^(self, other))
793     }
794 
795     fn not(self) -> ValueResult<Self> {
796         Ok(match self {
797             DataValue::I8(a) => DataValue::I8(!a),
798             DataValue::I16(a) => DataValue::I16(!a),
799             DataValue::I32(a) => DataValue::I32(!a),
800             DataValue::I64(a) => DataValue::I64(!a),
801             DataValue::I128(a) => DataValue::I128(!a),
802             DataValue::F32(a) => DataValue::F32(!a),
803             DataValue::F64(a) => DataValue::F64(!a),
804             DataValue::V128(a) => {
805                 let mut a2 = a.clone();
806                 for a in a2.iter_mut() {
807                     *a = !*a;
808                 }
809                 DataValue::V128(a2)
810             }
811             _ => unimplemented!(),
812         })
813     }
814 
815     fn count_ones(self) -> ValueResult<Self> {
816         unary_match!(count_ones(&self); [I8, I16, I32, I64, I128]; [i8, i16, i32, i64, i128])
817     }
818 
819     fn leading_ones(self) -> ValueResult<Self> {
820         unary_match!(leading_ones(&self); [I8, I16, I32, I64, I128]; [i8, i16, i32, i64, i128])
821     }
822 
823     fn leading_zeros(self) -> ValueResult<Self> {
824         unary_match!(leading_zeros(&self); [I8, I16, I32, I64, I128]; [i8, i16, i32, i64, i128])
825     }
826 
827     fn trailing_zeros(self) -> ValueResult<Self> {
828         unary_match!(trailing_zeros(&self); [I8, I16, I32, I64, I128]; [i8, i16, i32, i64, i128])
829     }
830 
831     fn reverse_bits(self) -> ValueResult<Self> {
832         unary_match!(reverse_bits(&self); [I8, I16, I32, I64, I128])
833     }
834 
835     fn swap_bytes(self) -> ValueResult<Self> {
836         unary_match!(swap_bytes(&self); [I16, I32, I64, I128])
837     }
838 
839     fn iter_lanes(&self, ty: Type) -> ValueResult<DataValueIterator> {
840         DataValueIterator::new(self, ty)
841     }
842 }
843 
844 /// Iterator for DataValue's
845 pub struct DataValueIterator {
846     ty: Type,
847     v: SimdVec<DataValue>,
848     idx: usize,
849 }
850 
851 impl DataValueIterator {
852     fn new(dv: &DataValue, ty: Type) -> Result<Self, ValueError> {
853         match extractlanes(dv, ty) {
854             Ok(v) => return Ok(Self { ty, v, idx: 0 }),
855             Err(err) => return Err(err),
856         }
857     }
858 }
859 
860 impl Iterator for DataValueIterator {
861     type Item = DataValue;
862 
863     fn next(&mut self) -> Option<Self::Item> {
864         if self.idx >= self.ty.lane_count() as usize {
865             return None;
866         }
867 
868         let dv = self.v[self.idx].clone();
869         self.idx += 1;
870         Some(dv)
871     }
872 }
873 
874 #[cfg(test)]
875 mod test {
876     use super::*;
877 
878     #[test]
879     fn test_iterator_v128() {
880         let dv = DataValue::V128([99, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]);
881         assert_eq!(simd_sum(dv, types::I8X16), 219);
882     }
883 
884     #[test]
885     fn test_iterator_v128_empty() {
886         let dv = DataValue::V128([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]);
887         assert_eq!(simd_sum(dv, types::I8X16), 0);
888     }
889 
890     #[test]
891     fn test_iterator_v128_ones() {
892         let dv = DataValue::V128([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]);
893         assert_eq!(simd_sum(dv, types::I8X16), 16);
894     }
895 
896     #[test]
897     fn test_iterator_v64_empty() {
898         let dv = DataValue::V64([0, 0, 0, 0, 0, 0, 0, 0]);
899         assert_eq!(simd_sum(dv, types::I8X8), 0);
900     }
901     #[test]
902     fn test_iterator_v64_ones() {
903         let dv = DataValue::V64([1, 1, 1, 1, 1, 1, 1, 1]);
904         assert_eq!(simd_sum(dv, types::I8X8), 8);
905     }
906     #[test]
907     fn test_iterator_v64() {
908         let dv = DataValue::V64([10, 20, 30, 40, 50, 60, 70, 80]);
909         assert_eq!(simd_sum(dv, types::I8X8), 360);
910     }
911 
912     fn simd_sum(dv: DataValue, ty: types::Type) -> i128 {
913         let itr = dv.iter_lanes(ty).unwrap();
914 
915         itr.map(|e| {
916             if let Some(v) = e.into_int_signed().ok() {
917                 v
918             } else {
919                 0
920             }
921         })
922         .sum()
923     }
924 }
925