1 use crate::abi::{self, align_to, scratch, LocalSlot}; 2 use crate::codegen::{CodeGenContext, Emission, FuncEnv}; 3 use crate::isa::{ 4 reg::{writable, Reg, WritableReg}, 5 CallingConvention, 6 }; 7 use anyhow::Result; 8 use cranelift_codegen::{ 9 binemit::CodeOffset, 10 ir::{Endianness, LibCall, MemFlags, RelSourceLoc, SourceLoc, UserExternalNameRef}, 11 Final, MachBufferFinalized, MachLabel, 12 }; 13 use std::{fmt::Debug, ops::Range}; 14 use wasmtime_environ::PtrSize; 15 16 pub(crate) use cranelift_codegen::ir::TrapCode; 17 18 #[derive(Eq, PartialEq)] 19 pub(crate) enum DivKind { 20 /// Signed division. 21 Signed, 22 /// Unsigned division. 23 Unsigned, 24 } 25 26 /// Remainder kind. 27 #[derive(Copy, Clone)] 28 pub(crate) enum RemKind { 29 /// Signed remainder. 30 Signed, 31 /// Unsigned remainder. 32 Unsigned, 33 } 34 35 impl RemKind { 36 pub fn is_signed(&self) -> bool { 37 matches!(self, Self::Signed) 38 } 39 } 40 41 /// Kinds of vector min operation supported by WebAssembly. 42 pub(crate) enum V128MinKind { 43 /// 16 lanes of signed 8-bit integers. 44 I8x16S, 45 /// 16 lanes of unsigned 8-bit integers. 46 I8x16U, 47 /// 8 lanes of signed 16-bit integers. 48 I16x8S, 49 /// 8 lanes of unsigned 16-bit integers. 50 I16x8U, 51 /// 4 lanes of signed 32-bit integers. 52 I32x4S, 53 /// 4 lanes of unsigned 32-bit integers. 54 I32x4U, 55 } 56 57 /// Kinds of vector max operation supported by WebAssembly. 58 pub(crate) enum V128MaxKind { 59 /// 16 lanes of signed 8-bit integers. 60 I8x16S, 61 /// 16 lanes of unsigned 8-bit integers. 62 I8x16U, 63 /// 8 lanes of signed 16-bit integers. 64 I16x8S, 65 /// 8 lanes of unsigned 16-bit integers. 66 I16x8U, 67 /// 4 lanes of signed 32-bit integers. 68 I32x4S, 69 /// 4 lanes of unsigned 32-bit integers. 70 I32x4U, 71 } 72 73 #[derive(Eq, PartialEq)] 74 pub(crate) enum MulWideKind { 75 Signed, 76 Unsigned, 77 } 78 79 /// Type of operation for a read-modify-write instruction. 80 pub(crate) enum RmwOp { 81 Add, 82 Sub, 83 Xchg, 84 And, 85 Or, 86 Xor, 87 } 88 89 /// The direction to perform the memory move. 90 #[derive(Debug, Clone, Eq, PartialEq)] 91 pub(crate) enum MemMoveDirection { 92 /// From high memory addresses to low memory addresses. 93 /// Invariant: the source location is closer to the FP than the destination 94 /// location, which will be closer to the SP. 95 HighToLow, 96 /// From low memory addresses to high memory addresses. 97 /// Invariant: the source location is closer to the SP than the destination 98 /// location, which will be closer to the FP. 99 LowToHigh, 100 } 101 102 /// Classifies how to treat float-to-int conversions. 103 #[derive(Debug, Copy, Clone, Eq, PartialEq)] 104 pub(crate) enum TruncKind { 105 /// Saturating conversion. If the source value is greater than the maximum 106 /// value of the destination type, the result is clamped to the 107 /// destination maximum value. 108 Checked, 109 /// An exception is raised if the source value is greater than the maximum 110 /// value of the destination type. 111 Unchecked, 112 } 113 114 impl TruncKind { 115 /// Returns true if the truncation kind is checked. 116 pub(crate) fn is_checked(&self) -> bool { 117 *self == TruncKind::Checked 118 } 119 120 /// Returns `true` if the trunc kind is [`Unchecked`]. 121 /// 122 /// [`Unchecked`]: TruncKind::Unchecked 123 #[must_use] 124 pub(crate) fn is_unchecked(&self) -> bool { 125 matches!(self, Self::Unchecked) 126 } 127 } 128 129 /// Representation of the stack pointer offset. 130 #[derive(Copy, Clone, Eq, PartialEq, Debug, PartialOrd, Ord, Default)] 131 pub struct SPOffset(u32); 132 133 impl SPOffset { 134 pub fn from_u32(offs: u32) -> Self { 135 Self(offs) 136 } 137 138 pub fn as_u32(&self) -> u32 { 139 self.0 140 } 141 } 142 143 /// A stack slot. 144 #[derive(Debug, Clone, Copy, Eq, PartialEq)] 145 pub struct StackSlot { 146 /// The location of the slot, relative to the stack pointer. 147 pub offset: SPOffset, 148 /// The size of the slot, in bytes. 149 pub size: u32, 150 } 151 152 impl StackSlot { 153 pub fn new(offs: SPOffset, size: u32) -> Self { 154 Self { offset: offs, size } 155 } 156 } 157 158 /// Kinds of integer binary comparison in WebAssembly. The [`MacroAssembler`] 159 /// implementation for each ISA is responsible for emitting the correct 160 /// sequence of instructions when lowering to machine code. 161 #[derive(Debug, Clone, Copy, Eq, PartialEq)] 162 pub(crate) enum IntCmpKind { 163 /// Equal. 164 Eq, 165 /// Not equal. 166 Ne, 167 /// Signed less than. 168 LtS, 169 /// Unsigned less than. 170 LtU, 171 /// Signed greater than. 172 GtS, 173 /// Unsigned greater than. 174 GtU, 175 /// Signed less than or equal. 176 LeS, 177 /// Unsigned less than or equal. 178 LeU, 179 /// Signed greater than or equal. 180 GeS, 181 /// Unsigned greater than or equal. 182 GeU, 183 } 184 185 /// Kinds of float binary comparison in WebAssembly. The [`MacroAssembler`] 186 /// implementation for each ISA is responsible for emitting the correct 187 /// sequence of instructions when lowering code. 188 #[derive(Debug)] 189 pub(crate) enum FloatCmpKind { 190 /// Equal. 191 Eq, 192 /// Not equal. 193 Ne, 194 /// Less than. 195 Lt, 196 /// Greater than. 197 Gt, 198 /// Less than or equal. 199 Le, 200 /// Greater than or equal. 201 Ge, 202 } 203 204 /// Kinds of shifts in WebAssembly.The [`masm`] implementation for each ISA is 205 /// responsible for emitting the correct sequence of instructions when 206 /// lowering to machine code. 207 #[derive(Debug, Clone, Copy, Eq, PartialEq)] 208 pub(crate) enum ShiftKind { 209 /// Left shift. 210 Shl, 211 /// Signed right shift. 212 ShrS, 213 /// Unsigned right shift. 214 ShrU, 215 /// Left rotate. 216 Rotl, 217 /// Right rotate. 218 Rotr, 219 } 220 221 /// Kinds of extends in WebAssembly. Each MacroAssembler implementation 222 /// is responsible for emitting the correct sequence of instructions when 223 /// lowering to machine code. 224 #[derive(Copy, Clone)] 225 pub(crate) enum ExtendKind { 226 Signed(Extend<Signed>), 227 Unsigned(Extend<Zero>), 228 } 229 230 #[derive(Copy, Clone)] 231 pub(crate) enum Signed {} 232 #[derive(Copy, Clone)] 233 pub(crate) enum Zero {} 234 235 pub(crate) trait ExtendType {} 236 237 impl ExtendType for Signed {} 238 impl ExtendType for Zero {} 239 240 #[derive(Copy, Clone)] 241 pub(crate) enum Extend<T: ExtendType> { 242 /// 8 to 32 bit extend. 243 I32Extend8, 244 /// 16 to 32 bit extend. 245 I32Extend16, 246 /// 8 to 64 bit extend. 247 I64Extend8, 248 /// 16 to 64 bit extend. 249 I64Extend16, 250 /// 32 to 64 bit extend. 251 I64Extend32, 252 253 /// Variant to hold the kind of extend marker. 254 /// 255 /// This is `Signed` or `Zero`, that are empty enums, which means that this variant cannot be 256 /// constructed. 257 __Kind(T), 258 } 259 260 impl From<Extend<Zero>> for ExtendKind { 261 fn from(value: Extend<Zero>) -> Self { 262 ExtendKind::Unsigned(value) 263 } 264 } 265 266 impl<T: ExtendType> Extend<T> { 267 pub fn from_size(&self) -> OperandSize { 268 match self { 269 Extend::I32Extend8 | Extend::I64Extend8 => OperandSize::S8, 270 Extend::I32Extend16 | Extend::I64Extend16 => OperandSize::S16, 271 Extend::I64Extend32 => OperandSize::S32, 272 Extend::__Kind(_) => unreachable!(), 273 } 274 } 275 276 pub fn to_size(&self) -> OperandSize { 277 match self { 278 Extend::I32Extend8 | Extend::I32Extend16 => OperandSize::S32, 279 Extend::I64Extend8 | Extend::I64Extend16 | Extend::I64Extend32 => OperandSize::S64, 280 Extend::__Kind(_) => unreachable!(), 281 } 282 } 283 284 pub fn from_bits(&self) -> u8 { 285 self.from_size().num_bits() 286 } 287 288 pub fn to_bits(&self) -> u8 { 289 self.to_size().num_bits() 290 } 291 } 292 293 impl From<Extend<Signed>> for ExtendKind { 294 fn from(value: Extend<Signed>) -> Self { 295 ExtendKind::Signed(value) 296 } 297 } 298 299 impl ExtendKind { 300 pub fn signed(&self) -> bool { 301 match self { 302 Self::Signed(_) => true, 303 _ => false, 304 } 305 } 306 307 pub fn from_bits(&self) -> u8 { 308 match self { 309 Self::Signed(s) => s.from_bits(), 310 Self::Unsigned(u) => u.from_bits(), 311 } 312 } 313 314 pub fn to_bits(&self) -> u8 { 315 match self { 316 Self::Signed(s) => s.to_bits(), 317 Self::Unsigned(u) => u.to_bits(), 318 } 319 } 320 } 321 322 /// Kinds of vector load and extends in WebAssembly. Each MacroAssembler 323 /// implementation is responsible for emitting the correct sequence of 324 /// instructions when lowering to machine code. 325 #[derive(Copy, Clone)] 326 pub(crate) enum V128LoadExtendKind { 327 /// Sign extends eight 8 bit integers to eight 16 bit lanes. 328 E8x8S, 329 /// Zero extends eight 8 bit integers to eight 16 bit lanes. 330 E8x8U, 331 /// Sign extends four 16 bit integers to four 32 bit lanes. 332 E16x4S, 333 /// Zero extends four 16 bit integers to four 32 bit lanes. 334 E16x4U, 335 /// Sign extends two 32 bit integers to two 64 bit lanes. 336 E32x2S, 337 /// Zero extends two 32 bit integers to two 64 bit lanes. 338 E32x2U, 339 } 340 341 /// Kinds of splat loads supported by WebAssembly. 342 pub(crate) enum SplatLoadKind { 343 /// 8 bits. 344 S8, 345 /// 16 bits. 346 S16, 347 /// 32 bits. 348 S32, 349 /// 64 bits. 350 S64, 351 } 352 353 /// Kinds of splat supported by WebAssembly. 354 #[derive(Copy, Debug, Clone, Eq, PartialEq)] 355 pub(crate) enum SplatKind { 356 /// 8 bit integer. 357 I8x16, 358 /// 16 bit integer. 359 I16x8, 360 /// 32 bit integer. 361 I32x4, 362 /// 64 bit integer. 363 I64x2, 364 /// 32 bit float. 365 F32x4, 366 /// 64 bit float. 367 F64x2, 368 } 369 370 impl SplatKind { 371 /// The lane size to use for different kinds of splats. 372 pub(crate) fn lane_size(&self) -> OperandSize { 373 match self { 374 SplatKind::I8x16 => OperandSize::S8, 375 SplatKind::I16x8 => OperandSize::S16, 376 SplatKind::I32x4 | SplatKind::F32x4 => OperandSize::S32, 377 SplatKind::I64x2 | SplatKind::F64x2 => OperandSize::S64, 378 } 379 } 380 } 381 382 /// Kinds of extract lane supported by WebAssembly. 383 #[derive(Copy, Debug, Clone, Eq, PartialEq)] 384 pub(crate) enum ExtractLaneKind { 385 /// 16 lanes of 8-bit integers sign extended to 32-bits. 386 I8x16S, 387 /// 16 lanes of 8-bit integers zero extended to 32-bits. 388 I8x16U, 389 /// 8 lanes of 16-bit integers sign extended to 32-bits. 390 I16x8S, 391 /// 8 lanes of 16-bit integers zero extended to 32-bits. 392 I16x8U, 393 /// 4 lanes of 32-bit integers. 394 I32x4, 395 /// 2 lanes of 64-bit integers. 396 I64x2, 397 /// 4 lanes of 32-bit floats. 398 F32x4, 399 /// 2 lanes of 64-bit floats. 400 F64x2, 401 } 402 403 impl ExtractLaneKind { 404 /// The lane size to use for different kinds of extract lane kinds. 405 pub(crate) fn lane_size(&self) -> OperandSize { 406 match self { 407 ExtractLaneKind::I8x16S | ExtractLaneKind::I8x16U => OperandSize::S8, 408 ExtractLaneKind::I16x8S | ExtractLaneKind::I16x8U => OperandSize::S16, 409 ExtractLaneKind::I32x4 | ExtractLaneKind::F32x4 => OperandSize::S32, 410 ExtractLaneKind::I64x2 | ExtractLaneKind::F64x2 => OperandSize::S64, 411 } 412 } 413 } 414 415 impl From<ExtractLaneKind> for Extend<Signed> { 416 fn from(value: ExtractLaneKind) -> Self { 417 match value { 418 ExtractLaneKind::I8x16S => Extend::I32Extend8, 419 ExtractLaneKind::I16x8S => Extend::I32Extend16, 420 _ => unimplemented!(), 421 } 422 } 423 } 424 425 /// Kinds of replace lane supported by WebAssembly. 426 pub(crate) enum ReplaceLaneKind { 427 /// 16 lanes of 8 bit integers. 428 I8x16, 429 /// 8 lanes of 16 bit integers. 430 I16x8, 431 /// 4 lanes of 32 bit integers. 432 I32x4, 433 /// 2 lanes of 64 bit integers. 434 I64x2, 435 /// 4 lanes of 32 bit floats. 436 F32x4, 437 /// 2 lanes of 64 bit floats. 438 F64x2, 439 } 440 441 impl ReplaceLaneKind { 442 /// The lane size to use for different kinds of replace lane kinds. 443 pub(crate) fn lane_size(&self) -> OperandSize { 444 match self { 445 ReplaceLaneKind::I8x16 => OperandSize::S8, 446 ReplaceLaneKind::I16x8 => OperandSize::S16, 447 ReplaceLaneKind::I32x4 => OperandSize::S32, 448 ReplaceLaneKind::I64x2 => OperandSize::S64, 449 ReplaceLaneKind::F32x4 => OperandSize::S32, 450 ReplaceLaneKind::F64x2 => OperandSize::S64, 451 } 452 } 453 } 454 455 /// Kinds of behavior supported by Wasm loads. 456 pub(crate) enum LoadKind { 457 /// Load the entire bytes of the operand size without any modifications. 458 Operand(OperandSize), 459 /// Atomic load, with optional scalar extend. 460 Atomic(OperandSize, Option<ExtendKind>), 461 /// Duplicate value into vector lanes. 462 Splat(SplatLoadKind), 463 /// Scalar (non-vector) extend. 464 ScalarExtend(ExtendKind), 465 /// Vector extend. 466 VectorExtend(V128LoadExtendKind), 467 /// Load content into select lane. 468 VectorLane(LaneSelector), 469 } 470 471 impl LoadKind { 472 /// Returns the [`OperandSize`] used in the load operation. 473 pub(crate) fn derive_operand_size(&self) -> OperandSize { 474 match self { 475 Self::ScalarExtend(extend) | Self::Atomic(_, Some(extend)) => { 476 Self::operand_size_for_scalar(extend) 477 } 478 Self::VectorExtend(_) => OperandSize::S64, 479 Self::Splat(kind) => Self::operand_size_for_splat(kind), 480 Self::Operand(size) 481 | Self::Atomic(size, None) 482 | Self::VectorLane(LaneSelector { size, .. }) => *size, 483 } 484 } 485 486 pub fn vector_lane(lane: u8, size: OperandSize) -> Self { 487 Self::VectorLane(LaneSelector { lane, size }) 488 } 489 490 fn operand_size_for_scalar(extend_kind: &ExtendKind) -> OperandSize { 491 match extend_kind { 492 ExtendKind::Signed(s) => s.from_size(), 493 ExtendKind::Unsigned(u) => u.from_size(), 494 } 495 } 496 497 fn operand_size_for_splat(kind: &SplatLoadKind) -> OperandSize { 498 match kind { 499 SplatLoadKind::S8 => OperandSize::S8, 500 SplatLoadKind::S16 => OperandSize::S16, 501 SplatLoadKind::S32 => OperandSize::S32, 502 SplatLoadKind::S64 => OperandSize::S64, 503 } 504 } 505 506 pub(crate) fn is_atomic(&self) -> bool { 507 matches!(self, Self::Atomic(_, _)) 508 } 509 } 510 511 /// Kinds of behavior supported by Wasm loads. 512 #[derive(Copy, Clone)] 513 pub enum StoreKind { 514 /// Store the entire bytes of the operand size without any modifications. 515 Operand(OperandSize), 516 /// Store the entire bytes of the operand size without any modifications, atomically. 517 Atomic(OperandSize), 518 /// Store the content of selected lane. 519 VectorLane(LaneSelector), 520 } 521 522 impl StoreKind { 523 pub fn vector_lane(lane: u8, size: OperandSize) -> Self { 524 Self::VectorLane(LaneSelector { lane, size }) 525 } 526 } 527 528 #[derive(Copy, Clone)] 529 pub struct LaneSelector { 530 pub lane: u8, 531 pub size: OperandSize, 532 } 533 534 /// Types of vector integer to float conversions supported by WebAssembly. 535 pub(crate) enum V128ConvertKind { 536 /// 4 lanes of signed 32-bit integers to 4 lanes of 32-bit floats. 537 I32x4S, 538 /// 4 lanes of unsigned 32-bit integers to 4 lanes of 32-bit floats. 539 I32x4U, 540 /// 4 lanes of signed 32-bit integers to low bits of 2 lanes of 64-bit 541 /// floats. 542 I32x4LowS, 543 /// 4 lanes of unsigned 32-bit integers to low bits of 2 lanes of 64-bit 544 /// floats. 545 I32x4LowU, 546 } 547 548 impl V128ConvertKind { 549 pub(crate) fn src_lane_size(&self) -> OperandSize { 550 match self { 551 V128ConvertKind::I32x4S 552 | V128ConvertKind::I32x4U 553 | V128ConvertKind::I32x4LowS 554 | V128ConvertKind::I32x4LowU => OperandSize::S32, 555 } 556 } 557 558 pub(crate) fn dst_lane_size(&self) -> OperandSize { 559 match self { 560 V128ConvertKind::I32x4S | V128ConvertKind::I32x4U => OperandSize::S32, 561 V128ConvertKind::I32x4LowS | V128ConvertKind::I32x4LowU => OperandSize::S64, 562 } 563 } 564 } 565 566 /// Kinds of vector narrowing operations supported by WebAssembly. 567 pub(crate) enum V128NarrowKind { 568 /// Narrow 8 lanes of 16-bit integers to 16 lanes of 8-bit integers using 569 /// signed saturation. 570 I16x8S, 571 /// Narrow 8 lanes of 16-bit integers to 16 lanes of 8-bit integers using 572 /// unsigned saturation. 573 I16x8U, 574 /// Narrow 4 lanes of 32-bit integers to 8 lanes of 16-bit integers using 575 /// signed saturation. 576 I32x4S, 577 /// Narrow 4 lanes of 32-bit integers to 8 lanes of 16-bit integers using 578 /// unsigned saturation. 579 I32x4U, 580 } 581 582 impl V128NarrowKind { 583 /// Return the size of the destination lanes. 584 pub(crate) fn dst_lane_size(&self) -> OperandSize { 585 match self { 586 Self::I16x8S | Self::I16x8U => OperandSize::S8, 587 Self::I32x4S | Self::I32x4U => OperandSize::S16, 588 } 589 } 590 } 591 592 /// Kinds of vector extending operations supported by WebAssembly. 593 #[derive(Debug, Copy, Clone)] 594 pub(crate) enum V128ExtendKind { 595 /// Low half of i8x16 sign extended. 596 LowI8x16S, 597 /// High half of i8x16 sign extended. 598 HighI8x16S, 599 /// Low half of i8x16 zero extended. 600 LowI8x16U, 601 /// High half of i8x16 zero extended. 602 HighI8x16U, 603 /// Low half of i16x8 sign extended. 604 LowI16x8S, 605 /// High half of i16x8 sign extended. 606 HighI16x8S, 607 /// Low half of i16x8 zero extended. 608 LowI16x8U, 609 /// High half of i16x8 zero extended. 610 HighI16x8U, 611 /// Low half of i32x4 sign extended. 612 LowI32x4S, 613 /// High half of i32x4 sign extended. 614 HighI32x4S, 615 /// Low half of i32x4 zero extended. 616 LowI32x4U, 617 /// High half of i32x4 zero extended. 618 HighI32x4U, 619 } 620 621 impl V128ExtendKind { 622 /// The size of the source's lanes. 623 pub(crate) fn src_lane_size(&self) -> OperandSize { 624 match self { 625 Self::LowI8x16S | Self::LowI8x16U | Self::HighI8x16S | Self::HighI8x16U => { 626 OperandSize::S8 627 } 628 Self::LowI16x8S | Self::LowI16x8U | Self::HighI16x8S | Self::HighI16x8U => { 629 OperandSize::S16 630 } 631 Self::LowI32x4S | Self::LowI32x4U | Self::HighI32x4S | Self::HighI32x4U => { 632 OperandSize::S32 633 } 634 } 635 } 636 } 637 638 /// Kinds of vector equalities and non-equalities supported by WebAssembly. 639 pub(crate) enum VectorEqualityKind { 640 /// 16 lanes of 8 bit integers. 641 I8x16, 642 /// 8 lanes of 16 bit integers. 643 I16x8, 644 /// 4 lanes of 32 bit integers. 645 I32x4, 646 /// 2 lanes of 64 bit integers. 647 I64x2, 648 /// 4 lanes of 32 bit floats. 649 F32x4, 650 /// 2 lanes of 64 bit floats. 651 F64x2, 652 } 653 654 impl VectorEqualityKind { 655 /// Get the lane size to use. 656 pub(crate) fn lane_size(&self) -> OperandSize { 657 match self { 658 Self::I8x16 => OperandSize::S8, 659 Self::I16x8 => OperandSize::S16, 660 Self::I32x4 | Self::F32x4 => OperandSize::S32, 661 Self::I64x2 | Self::F64x2 => OperandSize::S64, 662 } 663 } 664 } 665 666 /// Kinds of vector comparisons supported by WebAssembly. 667 pub(crate) enum VectorCompareKind { 668 /// 16 lanes of signed 8 bit integers. 669 I8x16S, 670 /// 16 lanes of unsigned 8 bit integers. 671 I8x16U, 672 /// 8 lanes of signed 16 bit integers. 673 I16x8S, 674 /// 8 lanes of unsigned 16 bit integers. 675 I16x8U, 676 /// 4 lanes of signed 32 bit integers. 677 I32x4S, 678 /// 4 lanes of unsigned 32 bit integers. 679 I32x4U, 680 /// 2 lanes of signed 64 bit integers. 681 I64x2S, 682 /// 4 lanes of 32 bit floats. 683 F32x4, 684 /// 2 lanes of 64 bit floats. 685 F64x2, 686 } 687 688 impl VectorCompareKind { 689 /// Get the lane size to use. 690 pub(crate) fn lane_size(&self) -> OperandSize { 691 match self { 692 Self::I8x16S | Self::I8x16U => OperandSize::S8, 693 Self::I16x8S | Self::I16x8U => OperandSize::S16, 694 Self::I32x4S | Self::I32x4U | Self::F32x4 => OperandSize::S32, 695 Self::I64x2S | Self::F64x2 => OperandSize::S64, 696 } 697 } 698 } 699 700 /// Kinds of vector absolute operations supported by WebAssembly. 701 #[derive(Copy, Debug, Clone, Eq, PartialEq)] 702 pub(crate) enum V128AbsKind { 703 /// 8 bit integers. 704 I8x16, 705 /// 16 bit integers. 706 I16x8, 707 /// 32 bit integers. 708 I32x4, 709 /// 64 bit integers. 710 I64x2, 711 /// 32 bit floats. 712 F32x4, 713 /// 64 bit floats. 714 F64x2, 715 } 716 717 impl V128AbsKind { 718 /// The lane size to use. 719 pub(crate) fn lane_size(&self) -> OperandSize { 720 match self { 721 Self::I8x16 => OperandSize::S8, 722 Self::I16x8 => OperandSize::S16, 723 Self::I32x4 | Self::F32x4 => OperandSize::S32, 724 Self::I64x2 | Self::F64x2 => OperandSize::S64, 725 } 726 } 727 } 728 729 /// Kinds of truncation for vectors supported by WebAssembly. 730 pub(crate) enum V128TruncKind { 731 /// Truncates 4 lanes of 32-bit floats to nearest integral value. 732 F32x4, 733 /// Truncates 2 lanes of 64-bit floats to nearest integral value. 734 F64x2, 735 /// Integers from signed F32x4. 736 I32x4FromF32x4S, 737 /// Integers from unsigned F32x4. 738 I32x4FromF32x4U, 739 /// Integers from signed F64x2. 740 I32x4FromF64x2SZero, 741 /// Integers from unsigned F64x2. 742 I32x4FromF64x2UZero, 743 } 744 745 impl V128TruncKind { 746 /// The size of the source lanes. 747 pub(crate) fn src_lane_size(&self) -> OperandSize { 748 match self { 749 V128TruncKind::F32x4 750 | V128TruncKind::I32x4FromF32x4S 751 | V128TruncKind::I32x4FromF32x4U => OperandSize::S32, 752 V128TruncKind::F64x2 753 | V128TruncKind::I32x4FromF64x2SZero 754 | V128TruncKind::I32x4FromF64x2UZero => OperandSize::S64, 755 } 756 } 757 758 /// The size of the destination lanes. 759 pub(crate) fn dst_lane_size(&self) -> OperandSize { 760 if let V128TruncKind::F64x2 = self { 761 OperandSize::S64 762 } else { 763 OperandSize::S32 764 } 765 } 766 } 767 768 /// Kinds of vector addition supported by WebAssembly. 769 pub(crate) enum V128AddKind { 770 /// 16 lanes of 8-bit integers wrapping. 771 I8x16, 772 /// 16 lanes of 8-bit integers signed saturating. 773 I8x16SatS, 774 /// 16 lanes of 8-bit integers unsigned saturating. 775 I8x16SatU, 776 /// 8 lanes of 16-bit integers wrapping. 777 I16x8, 778 /// 8 lanes of 16-bit integers signed saturating. 779 I16x8SatS, 780 /// 8 lanes of 16-bit integers unsigned saturating. 781 I16x8SatU, 782 /// 4 lanes of 32-bit integers wrapping. 783 I32x4, 784 /// 2 lanes of 64-bit integers wrapping. 785 I64x2, 786 } 787 788 /// Kinds of vector subtraction supported by WebAssembly. 789 pub(crate) enum V128SubKind { 790 /// 16 lanes of 8-bit integers wrapping. 791 I8x16, 792 /// 16 lanes of 8-bit integers signed saturating. 793 I8x16SatS, 794 /// 16 lanes of 8-bit integers unsigned saturating. 795 I8x16SatU, 796 /// 8 lanes of 16-bit integers wrapping. 797 I16x8, 798 /// 8 lanes of 16-bit integers signed saturating. 799 I16x8SatS, 800 /// 8 lanes of 16-bit integers unsigned saturating. 801 I16x8SatU, 802 /// 4 lanes of 32-bit integers wrapping. 803 I32x4, 804 /// 2 lanes of 64-bit integers wrapping. 805 I64x2, 806 } 807 808 impl From<V128NegKind> for V128SubKind { 809 fn from(value: V128NegKind) -> Self { 810 match value { 811 V128NegKind::I8x16 => Self::I8x16, 812 V128NegKind::I16x8 => Self::I16x8, 813 V128NegKind::I32x4 => Self::I32x4, 814 V128NegKind::I64x2 => Self::I64x2, 815 } 816 } 817 } 818 819 /// Kinds of vector multiplication supported by WebAssembly. 820 pub(crate) enum V128MulKind { 821 /// 8 lanes of 16-bit integers. 822 I16x8, 823 /// 4 lanes of 32-bit integers. 824 I32x4, 825 /// 2 lanes of 64-bit integers. 826 I64x2, 827 } 828 829 /// Kinds of vector negation supported by WebAssembly. 830 pub(crate) enum V128NegKind { 831 /// 16 lanes of 8-bit integers. 832 I8x16, 833 /// 8 lanes of 16-bit integers. 834 I16x8, 835 /// 4 lanes of 32-bit integers. 836 I32x4, 837 /// 2 lanes of 64-bit integers. 838 I64x2, 839 } 840 841 /// Kinds of extended pairwise addition supported by WebAssembly. 842 pub(crate) enum V128ExtAddKind { 843 /// 16 lanes of signed 8-bit integers. 844 I8x16S, 845 /// 16 lanes of unsigned 8-bit integers. 846 I8x16U, 847 /// 8 lanes of signed 16-bit integers. 848 I16x8S, 849 /// 8 lanes of unsigned 16-bit integers. 850 I16x8U, 851 } 852 853 impl From<V128ExtAddKind> for V128AddKind { 854 fn from(value: V128ExtAddKind) -> Self { 855 match value { 856 V128ExtAddKind::I8x16S | V128ExtAddKind::I8x16U => Self::I16x8, 857 V128ExtAddKind::I16x8S | V128ExtAddKind::I16x8U => Self::I32x4, 858 } 859 } 860 } 861 862 /// Kinds of vector extended multiplication supported by WebAssembly. 863 #[derive(Debug, Clone, Copy)] 864 pub(crate) enum V128ExtMulKind { 865 LowI8x16S, 866 HighI8x16S, 867 LowI8x16U, 868 HighI8x16U, 869 LowI16x8S, 870 HighI16x8S, 871 LowI16x8U, 872 HighI16x8U, 873 LowI32x4S, 874 HighI32x4S, 875 LowI32x4U, 876 HighI32x4U, 877 } 878 879 impl From<V128ExtMulKind> for V128ExtendKind { 880 fn from(value: V128ExtMulKind) -> Self { 881 match value { 882 V128ExtMulKind::LowI8x16S => Self::LowI8x16S, 883 V128ExtMulKind::HighI8x16S => Self::HighI8x16S, 884 V128ExtMulKind::LowI8x16U => Self::LowI8x16U, 885 V128ExtMulKind::HighI8x16U => Self::HighI8x16U, 886 V128ExtMulKind::LowI16x8S => Self::LowI16x8S, 887 V128ExtMulKind::HighI16x8S => Self::HighI16x8S, 888 V128ExtMulKind::LowI16x8U => Self::LowI16x8U, 889 V128ExtMulKind::HighI16x8U => Self::HighI16x8U, 890 V128ExtMulKind::LowI32x4S => Self::LowI32x4S, 891 V128ExtMulKind::HighI32x4S => Self::HighI32x4S, 892 V128ExtMulKind::LowI32x4U => Self::LowI32x4U, 893 V128ExtMulKind::HighI32x4U => Self::HighI32x4U, 894 } 895 } 896 } 897 898 impl From<V128ExtMulKind> for V128MulKind { 899 fn from(value: V128ExtMulKind) -> Self { 900 match value { 901 V128ExtMulKind::LowI8x16S 902 | V128ExtMulKind::HighI8x16S 903 | V128ExtMulKind::LowI8x16U 904 | V128ExtMulKind::HighI8x16U => Self::I16x8, 905 V128ExtMulKind::LowI16x8S 906 | V128ExtMulKind::HighI16x8S 907 | V128ExtMulKind::LowI16x8U 908 | V128ExtMulKind::HighI16x8U => Self::I32x4, 909 V128ExtMulKind::LowI32x4S 910 | V128ExtMulKind::HighI32x4S 911 | V128ExtMulKind::LowI32x4U 912 | V128ExtMulKind::HighI32x4U => Self::I64x2, 913 } 914 } 915 } 916 917 /// Operand size, in bits. 918 #[derive(Copy, Debug, Clone, Eq, PartialEq)] 919 pub(crate) enum OperandSize { 920 /// 8 bits. 921 S8, 922 /// 16 bits. 923 S16, 924 /// 32 bits. 925 S32, 926 /// 64 bits. 927 S64, 928 /// 128 bits. 929 S128, 930 } 931 932 impl OperandSize { 933 /// The number of bits in the operand. 934 pub fn num_bits(&self) -> u8 { 935 match self { 936 OperandSize::S8 => 8, 937 OperandSize::S16 => 16, 938 OperandSize::S32 => 32, 939 OperandSize::S64 => 64, 940 OperandSize::S128 => 128, 941 } 942 } 943 944 /// The number of bytes in the operand. 945 pub fn bytes(&self) -> u32 { 946 match self { 947 Self::S8 => 1, 948 Self::S16 => 2, 949 Self::S32 => 4, 950 Self::S64 => 8, 951 Self::S128 => 16, 952 } 953 } 954 955 /// The binary logarithm of the number of bits in the operand. 956 pub fn log2(&self) -> u8 { 957 match self { 958 OperandSize::S8 => 3, 959 OperandSize::S16 => 4, 960 OperandSize::S32 => 5, 961 OperandSize::S64 => 6, 962 OperandSize::S128 => 7, 963 } 964 } 965 966 /// Create an [`OperandSize`] from the given number of bytes. 967 pub fn from_bytes(bytes: u8) -> Self { 968 use OperandSize::*; 969 match bytes { 970 4 => S32, 971 8 => S64, 972 16 => S128, 973 _ => panic!("Invalid bytes {bytes} for OperandSize"), 974 } 975 } 976 977 pub fn extend_to<T: ExtendType>(&self, to: Self) -> Option<Extend<T>> { 978 match to { 979 OperandSize::S32 => match self { 980 OperandSize::S8 => Some(Extend::I32Extend8), 981 OperandSize::S16 => Some(Extend::I32Extend16), 982 _ => None, 983 }, 984 OperandSize::S64 => match self { 985 OperandSize::S8 => Some(Extend::I64Extend8), 986 OperandSize::S16 => Some(Extend::I64Extend16), 987 OperandSize::S32 => Some(Extend::I64Extend32), 988 _ => None, 989 }, 990 _ => None, 991 } 992 } 993 } 994 995 /// An abstraction over a register or immediate. 996 #[derive(Copy, Clone, Debug, PartialEq, Eq)] 997 pub(crate) enum RegImm { 998 /// A register. 999 Reg(Reg), 1000 /// A tagged immediate argument. 1001 Imm(Imm), 1002 } 1003 1004 /// An tagged representation of an immediate. 1005 #[derive(Copy, Clone, Debug, PartialEq, Eq)] 1006 pub(crate) enum Imm { 1007 /// I32 immediate. 1008 I32(u32), 1009 /// I64 immediate. 1010 I64(u64), 1011 /// F32 immediate. 1012 F32(u32), 1013 /// F64 immediate. 1014 F64(u64), 1015 /// V128 immediate. 1016 V128(i128), 1017 } 1018 1019 impl Imm { 1020 /// Create a new I64 immediate. 1021 pub fn i64(val: i64) -> Self { 1022 Self::I64(val as u64) 1023 } 1024 1025 /// Create a new I32 immediate. 1026 pub fn i32(val: i32) -> Self { 1027 Self::I32(val as u32) 1028 } 1029 1030 /// Create a new F32 immediate. 1031 pub fn f32(bits: u32) -> Self { 1032 Self::F32(bits) 1033 } 1034 1035 /// Create a new F64 immediate. 1036 pub fn f64(bits: u64) -> Self { 1037 Self::F64(bits) 1038 } 1039 1040 /// Create a new V128 immediate. 1041 pub fn v128(bits: i128) -> Self { 1042 Self::V128(bits) 1043 } 1044 1045 /// Convert the immediate to i32, if possible. 1046 pub fn to_i32(&self) -> Option<i32> { 1047 match self { 1048 Self::I32(v) => Some(*v as i32), 1049 Self::I64(v) => i32::try_from(*v as i64).ok(), 1050 _ => None, 1051 } 1052 } 1053 1054 /// Returns true if the [`Imm`] is float. 1055 pub fn is_float(&self) -> bool { 1056 match self { 1057 Self::F32(_) | Self::F64(_) => true, 1058 _ => false, 1059 } 1060 } 1061 1062 /// Get the operand size of the immediate. 1063 pub fn size(&self) -> OperandSize { 1064 match self { 1065 Self::I32(_) | Self::F32(_) => OperandSize::S32, 1066 Self::I64(_) | Self::F64(_) => OperandSize::S64, 1067 Self::V128(_) => OperandSize::S128, 1068 } 1069 } 1070 1071 /// Get a little endian representation of the immediate. 1072 /// 1073 /// This method heap allocates and is intended to be used when adding 1074 /// values to the constant pool. 1075 pub fn to_bytes(&self) -> Vec<u8> { 1076 match self { 1077 Imm::I32(n) => n.to_le_bytes().to_vec(), 1078 Imm::I64(n) => n.to_le_bytes().to_vec(), 1079 Imm::F32(n) => n.to_le_bytes().to_vec(), 1080 Imm::F64(n) => n.to_le_bytes().to_vec(), 1081 Imm::V128(n) => n.to_le_bytes().to_vec(), 1082 } 1083 } 1084 } 1085 1086 /// The location of the [VMcontext] used for function calls. 1087 #[derive(Copy, Clone, Debug, Eq, PartialEq)] 1088 pub(crate) enum VMContextLoc { 1089 /// Dynamic, stored in the given register. 1090 Reg(Reg), 1091 /// The pinned [VMContext] register. 1092 Pinned, 1093 } 1094 1095 /// The maximum number of context arguments currently used across the compiler. 1096 pub(crate) const MAX_CONTEXT_ARGS: usize = 2; 1097 1098 /// Out-of-band special purpose arguments used for function call emission. 1099 /// 1100 /// We cannot rely on the value stack for these values given that inserting 1101 /// register or memory values at arbitrary locations of the value stack has the 1102 /// potential to break the stack ordering principle, which states that older 1103 /// values must always precede newer values, effectively simulating the order of 1104 /// values in the machine stack. 1105 /// The [ContextArgs] are meant to be resolved at every callsite; in some cases 1106 /// it might be possible to construct it early on, but given that it might 1107 /// contain allocatable registers, it's preferred to construct it in 1108 /// [FnCall::emit]. 1109 #[derive(Clone, Debug)] 1110 pub(crate) enum ContextArgs { 1111 /// No context arguments required. This is used for libcalls that don't 1112 /// require any special context arguments. For example builtin functions 1113 /// that perform float calculations. 1114 None, 1115 /// A single context argument is required; the current pinned [VMcontext] 1116 /// register must be passed as the first argument of the function call. 1117 VMContext([VMContextLoc; 1]), 1118 /// The callee and caller context arguments are required. In this case, the 1119 /// callee context argument is usually stored into an allocatable register 1120 /// and the caller is always the current pinned [VMContext] pointer. 1121 CalleeAndCallerVMContext([VMContextLoc; MAX_CONTEXT_ARGS]), 1122 } 1123 1124 impl ContextArgs { 1125 /// Construct an empty [ContextArgs]. 1126 pub fn none() -> Self { 1127 Self::None 1128 } 1129 1130 /// Construct a [ContextArgs] declaring the usage of the pinned [VMContext] 1131 /// register as both the caller and callee context arguments. 1132 pub fn pinned_callee_and_caller_vmctx() -> Self { 1133 Self::CalleeAndCallerVMContext([VMContextLoc::Pinned, VMContextLoc::Pinned]) 1134 } 1135 1136 /// Construct a [ContextArgs] that declares the usage of the pinned 1137 /// [VMContext] register as the only context argument. 1138 pub fn pinned_vmctx() -> Self { 1139 Self::VMContext([VMContextLoc::Pinned]) 1140 } 1141 1142 /// Construct a [ContextArgs] that declares a dynamic callee context and the 1143 /// pinned [VMContext] register as the context arguments. 1144 pub fn with_callee_and_pinned_caller(callee_vmctx: Reg) -> Self { 1145 Self::CalleeAndCallerVMContext([VMContextLoc::Reg(callee_vmctx), VMContextLoc::Pinned]) 1146 } 1147 1148 /// Get the length of the [ContextArgs]. 1149 pub fn len(&self) -> usize { 1150 self.as_slice().len() 1151 } 1152 1153 /// Get a slice of the context arguments. 1154 pub fn as_slice(&self) -> &[VMContextLoc] { 1155 match self { 1156 Self::None => &[], 1157 Self::VMContext(a) => a.as_slice(), 1158 Self::CalleeAndCallerVMContext(a) => a.as_slice(), 1159 } 1160 } 1161 } 1162 1163 #[derive(Copy, Clone, Debug)] 1164 pub(crate) enum CalleeKind { 1165 /// A function call to a raw address. 1166 Indirect(Reg), 1167 /// A function call to a local function. 1168 Direct(UserExternalNameRef), 1169 /// Call to a well known LibCall. 1170 LibCall(LibCall), 1171 } 1172 1173 impl CalleeKind { 1174 /// Creates a callee kind from a register. 1175 pub fn indirect(reg: Reg) -> Self { 1176 Self::Indirect(reg) 1177 } 1178 1179 /// Creates a direct callee kind from a function name. 1180 pub fn direct(name: UserExternalNameRef) -> Self { 1181 Self::Direct(name) 1182 } 1183 1184 /// Creates a known callee kind from a libcall. 1185 pub fn libcall(call: LibCall) -> Self { 1186 Self::LibCall(call) 1187 } 1188 } 1189 1190 impl RegImm { 1191 /// Register constructor. 1192 pub fn reg(r: Reg) -> Self { 1193 RegImm::Reg(r) 1194 } 1195 1196 /// I64 immediate constructor. 1197 pub fn i64(val: i64) -> Self { 1198 RegImm::Imm(Imm::i64(val)) 1199 } 1200 1201 /// I32 immediate constructor. 1202 pub fn i32(val: i32) -> Self { 1203 RegImm::Imm(Imm::i32(val)) 1204 } 1205 1206 /// F32 immediate, stored using its bits representation. 1207 pub fn f32(bits: u32) -> Self { 1208 RegImm::Imm(Imm::f32(bits)) 1209 } 1210 1211 /// F64 immediate, stored using its bits representation. 1212 pub fn f64(bits: u64) -> Self { 1213 RegImm::Imm(Imm::f64(bits)) 1214 } 1215 1216 /// V128 immediate. 1217 pub fn v128(bits: i128) -> Self { 1218 RegImm::Imm(Imm::v128(bits)) 1219 } 1220 } 1221 1222 impl From<Reg> for RegImm { 1223 fn from(r: Reg) -> Self { 1224 Self::Reg(r) 1225 } 1226 } 1227 1228 #[derive(Debug)] 1229 pub enum RoundingMode { 1230 Nearest, 1231 Up, 1232 Down, 1233 Zero, 1234 } 1235 1236 /// Memory flags for trusted loads/stores. 1237 pub const TRUSTED_FLAGS: MemFlags = MemFlags::trusted(); 1238 1239 /// Flags used for WebAssembly loads / stores. 1240 /// Untrusted by default so we don't set `no_trap`. 1241 /// We also ensure that the endianness is the right one for WebAssembly. 1242 pub const UNTRUSTED_FLAGS: MemFlags = MemFlags::new().with_endianness(Endianness::Little); 1243 1244 /// Generic MacroAssembler interface used by the code generation. 1245 /// 1246 /// The MacroAssembler trait aims to expose an interface, high-level enough, 1247 /// so that each ISA can provide its own lowering to machine code. For example, 1248 /// for WebAssembly operators that don't have a direct mapping to a machine 1249 /// a instruction, the interface defines a signature matching the WebAssembly 1250 /// operator, allowing each implementation to lower such operator entirely. 1251 /// This approach attributes more responsibility to the MacroAssembler, but frees 1252 /// the caller from concerning about assembling the right sequence of 1253 /// instructions at the operator callsite. 1254 /// 1255 /// The interface defaults to a three-argument form for binary operations; 1256 /// this allows a natural mapping to instructions for RISC architectures, 1257 /// that use three-argument form. 1258 /// This approach allows for a more general interface that can be restricted 1259 /// where needed, in the case of architectures that use a two-argument form. 1260 1261 pub(crate) trait MacroAssembler { 1262 /// The addressing mode. 1263 type Address: Copy + Debug; 1264 1265 /// The pointer representation of the target ISA, 1266 /// used to access information from [`VMOffsets`]. 1267 type Ptr: PtrSize; 1268 1269 /// The ABI details of the target. 1270 type ABI: abi::ABI; 1271 1272 /// Emit the function prologue. 1273 fn prologue(&mut self, vmctx: Reg) -> Result<()> { 1274 self.frame_setup()?; 1275 self.check_stack(vmctx) 1276 } 1277 1278 /// Generate the frame setup sequence. 1279 fn frame_setup(&mut self) -> Result<()>; 1280 1281 /// Generate the frame restore sequence. 1282 fn frame_restore(&mut self) -> Result<()>; 1283 1284 /// Emit a stack check. 1285 fn check_stack(&mut self, vmctx: Reg) -> Result<()>; 1286 1287 /// Emit the function epilogue. 1288 fn epilogue(&mut self) -> Result<()> { 1289 self.frame_restore() 1290 } 1291 1292 /// Reserve stack space. 1293 fn reserve_stack(&mut self, bytes: u32) -> Result<()>; 1294 1295 /// Free stack space. 1296 fn free_stack(&mut self, bytes: u32) -> Result<()>; 1297 1298 /// Reset the stack pointer to the given offset; 1299 /// 1300 /// Used to reset the stack pointer to a given offset 1301 /// when dealing with unreachable code. 1302 fn reset_stack_pointer(&mut self, offset: SPOffset) -> Result<()>; 1303 1304 /// Get the address of a local slot. 1305 fn local_address(&mut self, local: &LocalSlot) -> Result<Self::Address>; 1306 1307 /// Constructs an address with an offset that is relative to the 1308 /// current position of the stack pointer (e.g. [sp + (sp_offset - 1309 /// offset)]. 1310 fn address_from_sp(&self, offset: SPOffset) -> Result<Self::Address>; 1311 1312 /// Constructs an address with an offset that is absolute to the 1313 /// current position of the stack pointer (e.g. [sp + offset]. 1314 fn address_at_sp(&self, offset: SPOffset) -> Result<Self::Address>; 1315 1316 /// Alias for [`Self::address_at_reg`] using the VMContext register as 1317 /// a base. The VMContext register is derived from the ABI type that is 1318 /// associated to the MacroAssembler. 1319 fn address_at_vmctx(&self, offset: u32) -> Result<Self::Address>; 1320 1321 /// Construct an address that is absolute to the current position 1322 /// of the given register. 1323 fn address_at_reg(&self, reg: Reg, offset: u32) -> Result<Self::Address>; 1324 1325 /// Emit a function call to either a local or external function. 1326 fn call( 1327 &mut self, 1328 stack_args_size: u32, 1329 f: impl FnMut(&mut Self) -> Result<(CalleeKind, CallingConvention)>, 1330 ) -> Result<u32>; 1331 1332 /// Get stack pointer offset. 1333 fn sp_offset(&self) -> Result<SPOffset>; 1334 1335 /// Perform a stack store. 1336 fn store(&mut self, src: RegImm, dst: Self::Address, size: OperandSize) -> Result<()>; 1337 1338 /// Alias for `MacroAssembler::store` with the operand size corresponding 1339 /// to the pointer size of the target. 1340 fn store_ptr(&mut self, src: Reg, dst: Self::Address) -> Result<()>; 1341 1342 /// Perform a WebAssembly store. 1343 /// A WebAssembly store introduces several additional invariants compared to 1344 /// [Self::store], more precisely, it can implicitly trap, in certain 1345 /// circumstances, even if explicit bounds checks are elided, in that sense, 1346 /// we consider this type of load as untrusted. It can also differ with 1347 /// regards to the endianness depending on the target ISA. For this reason, 1348 /// [Self::wasm_store], should be explicitly used when emitting WebAssembly 1349 /// stores. 1350 fn wasm_store(&mut self, src: Reg, dst: Self::Address, store_kind: StoreKind) -> Result<()>; 1351 1352 /// Perform a zero-extended stack load. 1353 fn load(&mut self, src: Self::Address, dst: WritableReg, size: OperandSize) -> Result<()>; 1354 1355 /// Perform a WebAssembly load. 1356 /// A WebAssembly load introduces several additional invariants compared to 1357 /// [Self::load], more precisely, it can implicitly trap, in certain 1358 /// circumstances, even if explicit bounds checks are elided, in that sense, 1359 /// we consider this type of load as untrusted. It can also differ with 1360 /// regards to the endianness depending on the target ISA. For this reason, 1361 /// [Self::wasm_load], should be explicitly used when emitting WebAssembly 1362 /// loads. 1363 fn wasm_load(&mut self, src: Self::Address, dst: WritableReg, kind: LoadKind) -> Result<()>; 1364 1365 /// Alias for `MacroAssembler::load` with the operand size corresponding 1366 /// to the pointer size of the target. 1367 fn load_ptr(&mut self, src: Self::Address, dst: WritableReg) -> Result<()>; 1368 1369 /// Loads the effective address into destination. 1370 fn load_addr( 1371 &mut self, 1372 _src: Self::Address, 1373 _dst: WritableReg, 1374 _size: OperandSize, 1375 ) -> Result<()>; 1376 1377 /// Pop a value from the machine stack into the given register. 1378 fn pop(&mut self, dst: WritableReg, size: OperandSize) -> Result<()>; 1379 1380 /// Perform a move. 1381 fn mov(&mut self, dst: WritableReg, src: RegImm, size: OperandSize) -> Result<()>; 1382 1383 /// Perform a conditional move. 1384 fn cmov(&mut self, dst: WritableReg, src: Reg, cc: IntCmpKind, size: OperandSize) 1385 -> Result<()>; 1386 1387 /// Performs a memory move of bytes from src to dest. 1388 /// Bytes are moved in blocks of 8 bytes, where possible. 1389 fn memmove( 1390 &mut self, 1391 src: SPOffset, 1392 dst: SPOffset, 1393 bytes: u32, 1394 direction: MemMoveDirection, 1395 ) -> Result<()> { 1396 match direction { 1397 MemMoveDirection::LowToHigh => debug_assert!(dst.as_u32() < src.as_u32()), 1398 MemMoveDirection::HighToLow => debug_assert!(dst.as_u32() > src.as_u32()), 1399 } 1400 // At least 4 byte aligned. 1401 debug_assert!(bytes % 4 == 0); 1402 let mut remaining = bytes; 1403 let word_bytes = <Self::ABI as abi::ABI>::word_bytes(); 1404 let scratch = scratch!(Self); 1405 1406 let mut dst_offs = dst.as_u32() - bytes; 1407 let mut src_offs = src.as_u32() - bytes; 1408 1409 let word_bytes = word_bytes as u32; 1410 while remaining >= word_bytes { 1411 remaining -= word_bytes; 1412 dst_offs += word_bytes; 1413 src_offs += word_bytes; 1414 1415 self.load_ptr( 1416 self.address_from_sp(SPOffset::from_u32(src_offs))?, 1417 writable!(scratch), 1418 )?; 1419 self.store_ptr( 1420 scratch.into(), 1421 self.address_from_sp(SPOffset::from_u32(dst_offs))?, 1422 )?; 1423 } 1424 1425 if remaining > 0 { 1426 let half_word = word_bytes / 2; 1427 let ptr_size = OperandSize::from_bytes(half_word as u8); 1428 debug_assert!(remaining == half_word); 1429 dst_offs += half_word; 1430 src_offs += half_word; 1431 1432 self.load( 1433 self.address_from_sp(SPOffset::from_u32(src_offs))?, 1434 writable!(scratch), 1435 ptr_size, 1436 )?; 1437 self.store( 1438 scratch.into(), 1439 self.address_from_sp(SPOffset::from_u32(dst_offs))?, 1440 ptr_size, 1441 )?; 1442 } 1443 Ok(()) 1444 } 1445 1446 /// Perform add operation. 1447 fn add(&mut self, dst: WritableReg, lhs: Reg, rhs: RegImm, size: OperandSize) -> Result<()>; 1448 1449 /// Perform a checked unsigned integer addition, emitting the provided trap 1450 /// if the addition overflows. 1451 fn checked_uadd( 1452 &mut self, 1453 dst: WritableReg, 1454 lhs: Reg, 1455 rhs: RegImm, 1456 size: OperandSize, 1457 trap: TrapCode, 1458 ) -> Result<()>; 1459 1460 /// Perform subtraction operation. 1461 fn sub(&mut self, dst: WritableReg, lhs: Reg, rhs: RegImm, size: OperandSize) -> Result<()>; 1462 1463 /// Perform multiplication operation. 1464 fn mul(&mut self, dst: WritableReg, lhs: Reg, rhs: RegImm, size: OperandSize) -> Result<()>; 1465 1466 /// Perform a floating point add operation. 1467 fn float_add(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) -> Result<()>; 1468 1469 /// Perform a floating point subtraction operation. 1470 fn float_sub(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) -> Result<()>; 1471 1472 /// Perform a floating point multiply operation. 1473 fn float_mul(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) -> Result<()>; 1474 1475 /// Perform a floating point divide operation. 1476 fn float_div(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) -> Result<()>; 1477 1478 /// Perform a floating point minimum operation. In x86, this will emit 1479 /// multiple instructions. 1480 fn float_min(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) -> Result<()>; 1481 1482 /// Perform a floating point maximum operation. In x86, this will emit 1483 /// multiple instructions. 1484 fn float_max(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, size: OperandSize) -> Result<()>; 1485 1486 /// Perform a floating point copysign operation. In x86, this will emit 1487 /// multiple instructions. 1488 fn float_copysign( 1489 &mut self, 1490 dst: WritableReg, 1491 lhs: Reg, 1492 rhs: Reg, 1493 size: OperandSize, 1494 ) -> Result<()>; 1495 1496 /// Perform a floating point abs operation. 1497 fn float_abs(&mut self, dst: WritableReg, size: OperandSize) -> Result<()>; 1498 1499 /// Perform a floating point negation operation. 1500 fn float_neg(&mut self, dst: WritableReg, size: OperandSize) -> Result<()>; 1501 1502 /// Perform a floating point floor operation. 1503 fn float_round< 1504 F: FnMut(&mut FuncEnv<Self::Ptr>, &mut CodeGenContext<Emission>, &mut Self) -> Result<()>, 1505 >( 1506 &mut self, 1507 mode: RoundingMode, 1508 env: &mut FuncEnv<Self::Ptr>, 1509 context: &mut CodeGenContext<Emission>, 1510 size: OperandSize, 1511 fallback: F, 1512 ) -> Result<()>; 1513 1514 /// Perform a floating point square root operation. 1515 fn float_sqrt(&mut self, dst: WritableReg, src: Reg, size: OperandSize) -> Result<()>; 1516 1517 /// Perform logical and operation. 1518 fn and(&mut self, dst: WritableReg, lhs: Reg, rhs: RegImm, size: OperandSize) -> Result<()>; 1519 1520 /// Perform logical or operation. 1521 fn or(&mut self, dst: WritableReg, lhs: Reg, rhs: RegImm, size: OperandSize) -> Result<()>; 1522 1523 /// Perform logical exclusive or operation. 1524 fn xor(&mut self, dst: WritableReg, lhs: Reg, rhs: RegImm, size: OperandSize) -> Result<()>; 1525 1526 /// Perform a shift operation between a register and an immediate. 1527 fn shift_ir( 1528 &mut self, 1529 dst: WritableReg, 1530 imm: u64, 1531 lhs: Reg, 1532 kind: ShiftKind, 1533 size: OperandSize, 1534 ) -> Result<()>; 1535 1536 /// Perform a shift operation between two registers. 1537 /// This case is special in that some architectures have specific expectations 1538 /// regarding the location of the instruction arguments. To free the 1539 /// caller from having to deal with the architecture specific constraints 1540 /// we give this function access to the code generation context, allowing 1541 /// each implementation to decide the lowering path. 1542 fn shift( 1543 &mut self, 1544 context: &mut CodeGenContext<Emission>, 1545 kind: ShiftKind, 1546 size: OperandSize, 1547 ) -> Result<()>; 1548 1549 /// Perform division operation. 1550 /// Division is special in that some architectures have specific 1551 /// expectations regarding the location of the instruction 1552 /// arguments and regarding the location of the quotient / 1553 /// remainder. To free the caller from having to deal with the 1554 /// architecture specific constraints we give this function access 1555 /// to the code generation context, allowing each implementation 1556 /// to decide the lowering path. For cases in which division is a 1557 /// unconstrained binary operation, the caller can decide to use 1558 /// the `CodeGenContext::i32_binop` or `CodeGenContext::i64_binop` 1559 /// functions. 1560 fn div( 1561 &mut self, 1562 context: &mut CodeGenContext<Emission>, 1563 kind: DivKind, 1564 size: OperandSize, 1565 ) -> Result<()>; 1566 1567 /// Calculate remainder. 1568 fn rem( 1569 &mut self, 1570 context: &mut CodeGenContext<Emission>, 1571 kind: RemKind, 1572 size: OperandSize, 1573 ) -> Result<()>; 1574 1575 /// Compares `src1` against `src2` for the side effect of setting processor 1576 /// flags. 1577 /// 1578 /// Note that `src1` is the left-hand-side of the comparison and `src2` is 1579 /// the right-hand-side, so if testing `a < b` then `src1 == a` and 1580 /// `src2 == b` 1581 fn cmp(&mut self, src1: Reg, src2: RegImm, size: OperandSize) -> Result<()>; 1582 1583 /// Compare src and dst and put the result in dst. 1584 /// This function will potentially emit a series of instructions. 1585 /// 1586 /// The initial value in `dst` is the left-hand-side of the comparison and 1587 /// the initial value in `src` is the right-hand-side of the comparison. 1588 /// That means for `a < b` then `dst == a` and `src == b`. 1589 fn cmp_with_set( 1590 &mut self, 1591 dst: WritableReg, 1592 src: RegImm, 1593 kind: IntCmpKind, 1594 size: OperandSize, 1595 ) -> Result<()>; 1596 1597 /// Compare floats in src1 and src2 and put the result in dst. 1598 /// In x86, this will emit multiple instructions. 1599 fn float_cmp_with_set( 1600 &mut self, 1601 dst: WritableReg, 1602 src1: Reg, 1603 src2: Reg, 1604 kind: FloatCmpKind, 1605 size: OperandSize, 1606 ) -> Result<()>; 1607 1608 /// Count the number of leading zeroes in src and put the result in dst. 1609 /// In x64, this will emit multiple instructions if the `has_lzcnt` flag is 1610 /// false. 1611 fn clz(&mut self, dst: WritableReg, src: Reg, size: OperandSize) -> Result<()>; 1612 1613 /// Count the number of trailing zeroes in src and put the result in dst.masm 1614 /// In x64, this will emit multiple instructions if the `has_tzcnt` flag is 1615 /// false. 1616 fn ctz(&mut self, dst: WritableReg, src: Reg, size: OperandSize) -> Result<()>; 1617 1618 /// Push the register to the stack, returning the stack slot metadata. 1619 // NB 1620 // The stack alignment should not be assumed after any call to `push`, 1621 // unless explicitly aligned otherwise. Typically, stack alignment is 1622 // maintained at call sites and during the execution of 1623 // epilogues. 1624 fn push(&mut self, src: Reg, size: OperandSize) -> Result<StackSlot>; 1625 1626 /// Finalize the assembly and return the result. 1627 fn finalize(self, base: Option<SourceLoc>) -> Result<MachBufferFinalized<Final>>; 1628 1629 /// Zero a particular register. 1630 fn zero(&mut self, reg: WritableReg) -> Result<()>; 1631 1632 /// Count the number of 1 bits in src and put the result in dst. In x64, 1633 /// this will emit multiple instructions if the `has_popcnt` flag is false. 1634 fn popcnt(&mut self, context: &mut CodeGenContext<Emission>, size: OperandSize) -> Result<()>; 1635 1636 /// Converts an i64 to an i32 by discarding the high 32 bits. 1637 fn wrap(&mut self, dst: WritableReg, src: Reg) -> Result<()>; 1638 1639 /// Extends an integer of a given size to a larger size. 1640 fn extend(&mut self, dst: WritableReg, src: Reg, kind: ExtendKind) -> Result<()>; 1641 1642 /// Emits one or more instructions to perform a signed truncation of a 1643 /// float into an integer. 1644 fn signed_truncate( 1645 &mut self, 1646 dst: WritableReg, 1647 src: Reg, 1648 src_size: OperandSize, 1649 dst_size: OperandSize, 1650 kind: TruncKind, 1651 ) -> Result<()>; 1652 1653 /// Emits one or more instructions to perform an unsigned truncation of a 1654 /// float into an integer. 1655 fn unsigned_truncate( 1656 &mut self, 1657 context: &mut CodeGenContext<Emission>, 1658 src_size: OperandSize, 1659 dst_size: OperandSize, 1660 kind: TruncKind, 1661 ) -> Result<()>; 1662 1663 /// Emits one or more instructions to perform a signed convert of an 1664 /// integer into a float. 1665 fn signed_convert( 1666 &mut self, 1667 dst: WritableReg, 1668 src: Reg, 1669 src_size: OperandSize, 1670 dst_size: OperandSize, 1671 ) -> Result<()>; 1672 1673 /// Emits one or more instructions to perform an unsigned convert of an 1674 /// integer into a float. 1675 fn unsigned_convert( 1676 &mut self, 1677 dst: WritableReg, 1678 src: Reg, 1679 tmp_gpr: Reg, 1680 src_size: OperandSize, 1681 dst_size: OperandSize, 1682 ) -> Result<()>; 1683 1684 /// Reinterpret a float as an integer. 1685 fn reinterpret_float_as_int( 1686 &mut self, 1687 dst: WritableReg, 1688 src: Reg, 1689 size: OperandSize, 1690 ) -> Result<()>; 1691 1692 /// Reinterpret an integer as a float. 1693 fn reinterpret_int_as_float( 1694 &mut self, 1695 dst: WritableReg, 1696 src: Reg, 1697 size: OperandSize, 1698 ) -> Result<()>; 1699 1700 /// Demote an f64 to an f32. 1701 fn demote(&mut self, dst: WritableReg, src: Reg) -> Result<()>; 1702 1703 /// Promote an f32 to an f64. 1704 fn promote(&mut self, dst: WritableReg, src: Reg) -> Result<()>; 1705 1706 /// Zero a given memory range. 1707 /// 1708 /// The default implementation divides the given memory range 1709 /// into word-sized slots. Then it unrolls a series of store 1710 /// instructions, effectively assigning zero to each slot. 1711 fn zero_mem_range(&mut self, mem: &Range<u32>) -> Result<()> { 1712 let word_size = <Self::ABI as abi::ABI>::word_bytes() as u32; 1713 if mem.is_empty() { 1714 return Ok(()); 1715 } 1716 1717 let start = if mem.start % word_size == 0 { 1718 mem.start 1719 } else { 1720 // Ensure that the start of the range is at least 4-byte aligned. 1721 assert!(mem.start % 4 == 0); 1722 let start = align_to(mem.start, word_size); 1723 let addr: Self::Address = self.local_address(&LocalSlot::i32(start))?; 1724 self.store(RegImm::i32(0), addr, OperandSize::S32)?; 1725 // Ensure that the new start of the range, is word-size aligned. 1726 assert!(start % word_size == 0); 1727 start 1728 }; 1729 1730 let end = align_to(mem.end, word_size); 1731 let slots = (end - start) / word_size; 1732 1733 if slots == 1 { 1734 let slot = LocalSlot::i64(start + word_size); 1735 let addr: Self::Address = self.local_address(&slot)?; 1736 self.store(RegImm::i64(0), addr, OperandSize::S64)?; 1737 } else { 1738 // TODO 1739 // Add an upper bound to this generation; 1740 // given a considerably large amount of slots 1741 // this will be inefficient. 1742 let zero = scratch!(Self); 1743 self.zero(writable!(zero))?; 1744 let zero = RegImm::reg(zero); 1745 1746 for step in (start..end).into_iter().step_by(word_size as usize) { 1747 let slot = LocalSlot::i64(step + word_size); 1748 let addr: Self::Address = self.local_address(&slot)?; 1749 self.store(zero, addr, OperandSize::S64)?; 1750 } 1751 } 1752 1753 Ok(()) 1754 } 1755 1756 /// Generate a label. 1757 fn get_label(&mut self) -> Result<MachLabel>; 1758 1759 /// Bind the given label at the current code offset. 1760 fn bind(&mut self, label: MachLabel) -> Result<()>; 1761 1762 /// Conditional branch. 1763 /// 1764 /// Performs a comparison between the two operands, 1765 /// and immediately after emits a jump to the given 1766 /// label destination if the condition is met. 1767 fn branch( 1768 &mut self, 1769 kind: IntCmpKind, 1770 lhs: Reg, 1771 rhs: RegImm, 1772 taken: MachLabel, 1773 size: OperandSize, 1774 ) -> Result<()>; 1775 1776 /// Emits and unconditional jump to the given label. 1777 fn jmp(&mut self, target: MachLabel) -> Result<()>; 1778 1779 /// Emits a jump table sequence. The default label is specified as 1780 /// the last element of the targets slice. 1781 fn jmp_table(&mut self, targets: &[MachLabel], index: Reg, tmp: Reg) -> Result<()>; 1782 1783 /// Emit an unreachable code trap. 1784 fn unreachable(&mut self) -> Result<()>; 1785 1786 /// Emit an unconditional trap. 1787 fn trap(&mut self, code: TrapCode) -> Result<()>; 1788 1789 /// Traps if the condition code is met. 1790 fn trapif(&mut self, cc: IntCmpKind, code: TrapCode) -> Result<()>; 1791 1792 /// Trap if the source register is zero. 1793 fn trapz(&mut self, src: Reg, code: TrapCode) -> Result<()>; 1794 1795 /// Ensures that the stack pointer is correctly positioned before an unconditional 1796 /// jump according to the requirements of the destination target. 1797 fn ensure_sp_for_jump(&mut self, target: SPOffset) -> Result<()> { 1798 let bytes = self 1799 .sp_offset()? 1800 .as_u32() 1801 .checked_sub(target.as_u32()) 1802 .unwrap_or(0); 1803 1804 if bytes > 0 { 1805 self.free_stack(bytes)?; 1806 } 1807 1808 Ok(()) 1809 } 1810 1811 /// Mark the start of a source location returning the machine code offset 1812 /// and the relative source code location. 1813 fn start_source_loc(&mut self, loc: RelSourceLoc) -> Result<(CodeOffset, RelSourceLoc)>; 1814 1815 /// Mark the end of a source location. 1816 fn end_source_loc(&mut self) -> Result<()>; 1817 1818 /// The current offset, in bytes from the beginning of the function. 1819 fn current_code_offset(&self) -> Result<CodeOffset>; 1820 1821 /// Performs a 128-bit addition 1822 fn add128( 1823 &mut self, 1824 dst_lo: WritableReg, 1825 dst_hi: WritableReg, 1826 lhs_lo: Reg, 1827 lhs_hi: Reg, 1828 rhs_lo: Reg, 1829 rhs_hi: Reg, 1830 ) -> Result<()>; 1831 1832 /// Performs a 128-bit subtraction 1833 fn sub128( 1834 &mut self, 1835 dst_lo: WritableReg, 1836 dst_hi: WritableReg, 1837 lhs_lo: Reg, 1838 lhs_hi: Reg, 1839 rhs_lo: Reg, 1840 rhs_hi: Reg, 1841 ) -> Result<()>; 1842 1843 /// Performs a widening multiplication from two 64-bit operands into a 1844 /// 128-bit result. 1845 /// 1846 /// Note that some platforms require special handling of registers in this 1847 /// instruction (e.g. x64) so full access to `CodeGenContext` is provided. 1848 fn mul_wide(&mut self, context: &mut CodeGenContext<Emission>, kind: MulWideKind) 1849 -> Result<()>; 1850 1851 /// Takes the value in a src operand and replicates it across lanes of 1852 /// `size` in a destination result. 1853 fn splat(&mut self, context: &mut CodeGenContext<Emission>, size: SplatKind) -> Result<()>; 1854 1855 /// Performs a shuffle between two 128-bit vectors into a 128-bit result 1856 /// using lanes as a mask to select which indexes to copy. 1857 fn shuffle(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, lanes: [u8; 16]) -> Result<()>; 1858 1859 /// Performs a swizzle between two 128-bit vectors into a 128-bit result. 1860 fn swizzle(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg) -> Result<()>; 1861 1862 /// Performs the RMW `op` operation on the passed `addr`. 1863 /// 1864 /// The value *before* the operation was performed is written back to the `operand` register. 1865 fn atomic_rmw( 1866 &mut self, 1867 context: &mut CodeGenContext<Emission>, 1868 addr: Self::Address, 1869 size: OperandSize, 1870 op: RmwOp, 1871 flags: MemFlags, 1872 extend: Option<Extend<Zero>>, 1873 ) -> Result<()>; 1874 1875 /// Extracts the scalar value from `src` in `lane` to `dst`. 1876 fn extract_lane( 1877 &mut self, 1878 src: Reg, 1879 dst: WritableReg, 1880 lane: u8, 1881 kind: ExtractLaneKind, 1882 ) -> Result<()>; 1883 1884 /// Replaces the value in `lane` in `dst` with the value in `src`. 1885 fn replace_lane( 1886 &mut self, 1887 src: RegImm, 1888 dst: WritableReg, 1889 lane: u8, 1890 kind: ReplaceLaneKind, 1891 ) -> Result<()>; 1892 1893 /// Perform an atomic CAS (compare-and-swap) operation with the value at `addr`, and `expected` 1894 /// and `replacement` (at the top of the context's stack). 1895 /// 1896 /// This method takes the `CodeGenContext` as an arguments to accommodate architectures that 1897 /// expect parameters in specific registers. The context stack contains the `replacement`, 1898 /// and `expected` values in that order. The implementer is expected to push the value at 1899 /// `addr` before the update to the context's stack before returning. 1900 fn atomic_cas( 1901 &mut self, 1902 context: &mut CodeGenContext<Emission>, 1903 addr: Self::Address, 1904 size: OperandSize, 1905 flags: MemFlags, 1906 extend: Option<Extend<Zero>>, 1907 ) -> Result<()>; 1908 1909 /// Compares vector registers `lhs` and `rhs` for equality and puts the 1910 /// vector of results in `dst`. 1911 fn v128_eq( 1912 &mut self, 1913 dst: WritableReg, 1914 lhs: Reg, 1915 rhs: Reg, 1916 kind: VectorEqualityKind, 1917 ) -> Result<()>; 1918 1919 /// Compares vector registers `lhs` and `rhs` for inequality and puts the 1920 /// vector of results in `dst`. 1921 fn v128_ne( 1922 &mut self, 1923 dst: WritableReg, 1924 lhs: Reg, 1925 rhs: Reg, 1926 kind: VectorEqualityKind, 1927 ) -> Result<()>; 1928 1929 /// Performs a less than comparison with vector registers `lhs` and `rhs` 1930 /// and puts the vector of results in `dst`. 1931 fn v128_lt( 1932 &mut self, 1933 dst: WritableReg, 1934 lhs: Reg, 1935 rhs: Reg, 1936 kind: VectorCompareKind, 1937 ) -> Result<()>; 1938 1939 /// Performs a less than or equal comparison with vector registers `lhs` 1940 /// and `rhs` and puts the vector of results in `dst`. 1941 fn v128_le( 1942 &mut self, 1943 dst: WritableReg, 1944 lhs: Reg, 1945 rhs: Reg, 1946 kind: VectorCompareKind, 1947 ) -> Result<()>; 1948 1949 /// Performs a greater than comparison with vector registers `lhs` and 1950 /// `rhs` and puts the vector of results in `dst`. 1951 fn v128_gt( 1952 &mut self, 1953 dst: WritableReg, 1954 lhs: Reg, 1955 rhs: Reg, 1956 kind: VectorCompareKind, 1957 ) -> Result<()>; 1958 1959 /// Performs a greater than or equal comparison with vector registers `lhs` 1960 /// and `rhs` and puts the vector of results in `dst`. 1961 fn v128_ge( 1962 &mut self, 1963 dst: WritableReg, 1964 lhs: Reg, 1965 rhs: Reg, 1966 kind: VectorCompareKind, 1967 ) -> Result<()>; 1968 1969 /// Emit a memory fence. 1970 fn fence(&mut self) -> Result<()>; 1971 1972 /// Perform a logical `not` operation on the 128bits vector value in `dst`. 1973 fn v128_not(&mut self, dst: WritableReg) -> Result<()>; 1974 1975 /// Perform a logical `and` operation on `src1` and `src1`, both 128bits vector values, writing 1976 /// the result to `dst`. 1977 fn v128_and(&mut self, src1: Reg, src2: Reg, dst: WritableReg) -> Result<()>; 1978 1979 /// Perform a logical `and_not` operation on `src1` and `src1`, both 128bits vector values, writing 1980 /// the result to `dst`. 1981 /// 1982 /// `and_not` is not commutative: dst = !src1 & src2. 1983 fn v128_and_not(&mut self, src1: Reg, src2: Reg, dst: WritableReg) -> Result<()>; 1984 1985 /// Perform a logical `or` operation on `src1` and `src1`, both 128bits vector values, writing 1986 /// the result to `dst`. 1987 fn v128_or(&mut self, src1: Reg, src2: Reg, dst: WritableReg) -> Result<()>; 1988 1989 /// Perform a logical `xor` operation on `src1` and `src1`, both 128bits vector values, writing 1990 /// the result to `dst`. 1991 fn v128_xor(&mut self, src1: Reg, src2: Reg, dst: WritableReg) -> Result<()>; 1992 1993 /// Given two 128bits vectors `src1` and `src2`, and a 128bits bitmask `mask`, selects bits 1994 /// from `src1` when mask is 1, and from `src2` when mask is 0. 1995 /// 1996 /// This is equivalent to: `v128.or(v128.and(src1, mask), v128.and(src2, v128.not(mask)))`. 1997 fn v128_bitselect(&mut self, src1: Reg, src2: Reg, mask: Reg, dst: WritableReg) -> Result<()>; 1998 1999 /// If any bit in `src` is 1, set `dst` to 1, or 0 otherwise. 2000 fn v128_any_true(&mut self, src: Reg, dst: WritableReg) -> Result<()>; 2001 2002 /// Convert vector of integers to vector of floating points. 2003 fn v128_convert(&mut self, src: Reg, dst: WritableReg, kind: V128ConvertKind) -> Result<()>; 2004 2005 /// Convert two input vectors into a smaller lane vector by narrowing each 2006 /// lane. 2007 fn v128_narrow( 2008 &mut self, 2009 src1: Reg, 2010 src2: Reg, 2011 dst: WritableReg, 2012 kind: V128NarrowKind, 2013 ) -> Result<()>; 2014 2015 /// Converts a vector containing two 64-bit floating point lanes to two 2016 /// 32-bit floating point lanes and setting the two higher lanes to 0. 2017 fn v128_demote(&mut self, src: Reg, dst: WritableReg) -> Result<()>; 2018 2019 /// Converts a vector containing four 32-bit floating point lanes to two 2020 /// 64-bit floating point lanes. Only the two lower lanes are converted. 2021 fn v128_promote(&mut self, src: Reg, dst: WritableReg) -> Result<()>; 2022 2023 /// Converts low or high half of the smaller lane vector to a larger lane 2024 /// vector. 2025 fn v128_extend(&mut self, src: Reg, dst: WritableReg, kind: V128ExtendKind) -> Result<()>; 2026 2027 /// Perform a vector add between `lsh` and `rhs`, placing the result in 2028 /// `dst`. 2029 fn v128_add(&mut self, lhs: Reg, rhs: Reg, dst: WritableReg, kind: V128AddKind) -> Result<()>; 2030 2031 /// Perform a vector sub between `lhs` and `rhs`, placing the result in `dst`. 2032 fn v128_sub(&mut self, lhs: Reg, rhs: Reg, dst: WritableReg, kind: V128SubKind) -> Result<()>; 2033 2034 /// Perform a vector lane-wise mul between `lhs` and `rhs`, placing the result in `dst`. 2035 fn v128_mul(&mut self, context: &mut CodeGenContext<Emission>, kind: V128MulKind) 2036 -> Result<()>; 2037 2038 /// Perform an absolute operation on a vector. 2039 fn v128_abs(&mut self, src: Reg, dst: WritableReg, kind: V128AbsKind) -> Result<()>; 2040 2041 /// Vectorized negate of the content of `op`. 2042 fn v128_neg(&mut self, op: WritableReg, kind: V128NegKind) -> Result<()>; 2043 2044 /// Perform the shift operation specified by `kind`, by the shift amount specified by the 32-bit 2045 /// integer at the top the the stack, on the 128-bit vector specified by the second value 2046 /// from the top of the stack, interpreted as packed integers of size `lane_width`. 2047 /// 2048 /// The shift amount is taken modulo `lane_width`. 2049 fn v128_shift( 2050 &mut self, 2051 context: &mut CodeGenContext<Emission>, 2052 lane_width: OperandSize, 2053 kind: ShiftKind, 2054 ) -> Result<()>; 2055 2056 /// Perform a saturating integer q-format rounding multiplication. 2057 fn v128_q15mulr_sat_s( 2058 &mut self, 2059 lhs: Reg, 2060 rhs: Reg, 2061 dst: WritableReg, 2062 size: OperandSize, 2063 ) -> Result<()>; 2064 2065 /// Sets `dst` to 1 if all lanes in `src` are non-zero, sets `dst` to 0 2066 /// otherwise. 2067 fn v128_all_true(&mut self, src: Reg, dst: WritableReg, size: OperandSize) -> Result<()>; 2068 2069 /// Extracts the high bit of each lane in `src` and produces a scalar mask 2070 /// with all bits concatenated in `dst`. 2071 fn v128_bitmask(&mut self, src: Reg, dst: WritableReg, size: OperandSize) -> Result<()>; 2072 2073 /// Lanewise truncation operation. 2074 /// 2075 /// If using an integer kind of truncation, then this performs a lane-wise 2076 /// saturating conversion from float to integer using the IEEE 2077 /// `convertToIntegerTowardZero` function. If any input lane is NaN, the 2078 /// resulting lane is 0. If the rounded integer value of a lane is outside 2079 /// the range of the destination type, the result is saturated to the 2080 /// nearest representable integer value. 2081 fn v128_trunc( 2082 &mut self, 2083 context: &mut CodeGenContext<Emission>, 2084 kind: V128TruncKind, 2085 ) -> Result<()>; 2086 2087 /// Perform a lane-wise `min` operation between `src1` and `src2`. 2088 fn v128_min(&mut self, src1: Reg, src2: Reg, dst: WritableReg, kind: V128MinKind) 2089 -> Result<()>; 2090 2091 /// Perform a lane-wise `max` operation between `src1` and `src2`. 2092 fn v128_max(&mut self, src1: Reg, src2: Reg, dst: WritableReg, kind: V128MaxKind) 2093 -> Result<()>; 2094 2095 /// Perform the lane-wise integer extended multiplication producing twice wider result than the 2096 /// inputs. This is equivalent to an extend followed by a multiply. 2097 /// 2098 /// The extension to be performed is inferred from the `lane_width` and the `kind` of extmul, 2099 /// e.g, if `lane_width` is `S16`, and `kind` is `LowSigned`, then we sign-extend the lower 2100 /// 8bits of the 16bits lanes. 2101 fn v128_extmul( 2102 &mut self, 2103 context: &mut CodeGenContext<Emission>, 2104 kind: V128ExtMulKind, 2105 ) -> Result<()>; 2106 2107 /// Perform the lane-wise integer extended pairwise addition producing extended results (twice 2108 /// wider results than the inputs). 2109 fn v128_extadd_pairwise( 2110 &mut self, 2111 src: Reg, 2112 dst: WritableReg, 2113 kind: V128ExtAddKind, 2114 ) -> Result<()>; 2115 2116 /// Lane-wise multiply signed 16-bit integers in `lhs` and `rhs` and add 2117 /// adjacent pairs of the 32-bit results. 2118 fn v128_dot(&mut self, lhs: Reg, rhs: Reg, dst: WritableReg) -> Result<()>; 2119 2120 /// Count the number of bits set in each lane. 2121 fn v128_popcnt(&mut self, context: &mut CodeGenContext<Emission>) -> Result<()>; 2122 2123 /// Lane-wise rounding average of vectors of integers in `lhs` and `rhs` 2124 /// and put the results in `dst`. 2125 fn v128_avgr(&mut self, lhs: Reg, rhs: Reg, dst: WritableReg, size: OperandSize) -> Result<()>; 2126 2127 /// Lane-wise ceiling of vector of floats. 2128 fn v128_ceil(&mut self, src: Reg, dst: WritableReg, size: OperandSize) -> Result<()>; 2129 2130 /// Lane-wise flooring of vector of floats. 2131 fn v128_floor(&mut self, src: Reg, dst: WritableReg, size: OperandSize) -> Result<()>; 2132 2133 /// Lane-wise rounding to nearest integer for vector of floats. 2134 fn v128_nearest(&mut self, src: Reg, dst: WritableReg, size: OperandSize) -> Result<()>; 2135 } 2136