1 //! This module is the central place for machine code emission. 2 //! It defines an implementation of wasmparser's Visitor trait 3 //! for `CodeGen`; which defines a visitor per op-code, 4 //! which validates and dispatches to the corresponding 5 //! machine code emitter. 6 7 use crate::abi::ABI; 8 use crate::codegen::CodeGen; 9 use crate::codegen::ControlStackFrame; 10 use crate::masm::{CmpKind, DivKind, MacroAssembler, OperandSize, RegImm, RemKind, ShiftKind}; 11 use crate::stack::Val; 12 use wasmparser::{BlockType, VisitOperator}; 13 use wasmtime_environ::{FuncIndex, GlobalIndex, WasmType}; 14 15 /// A macro to define unsupported WebAssembly operators. 16 /// 17 /// This macro calls itself recursively; 18 /// 1. It no-ops when matching a supported operator. 19 /// 2. Defines the visitor function and panics when 20 /// matching an unsupported operator. 21 macro_rules! def_unsupported { 22 ($( @$proposal:ident $op:ident $({ $($arg:ident: $argty:ty),* })? => $visit:ident)*) => { 23 $( 24 def_unsupported!( 25 emit 26 $op 27 28 fn $visit(&mut self $($(,$arg: $argty)*)?) -> Self::Output { 29 $($(let _ = $arg;)*)? 30 todo!(stringify!($op)) 31 } 32 ); 33 )* 34 }; 35 36 (emit I32Const $($rest:tt)*) => {}; 37 (emit I64Const $($rest:tt)*) => {}; 38 (emit I32Add $($rest:tt)*) => {}; 39 (emit I64Add $($rest:tt)*) => {}; 40 (emit I32Sub $($rest:tt)*) => {}; 41 (emit I32Mul $($rest:tt)*) => {}; 42 (emit I32DivS $($rest:tt)*) => {}; 43 (emit I32DivU $($rest:tt)*) => {}; 44 (emit I64DivS $($rest:tt)*) => {}; 45 (emit I64DivU $($rest:tt)*) => {}; 46 (emit I64RemU $($rest:tt)*) => {}; 47 (emit I64RemS $($rest:tt)*) => {}; 48 (emit I32RemU $($rest:tt)*) => {}; 49 (emit I32RemS $($rest:tt)*) => {}; 50 (emit I64Mul $($rest:tt)*) => {}; 51 (emit I64Sub $($rest:tt)*) => {}; 52 (emit I32Eq $($rest:tt)*) => {}; 53 (emit I64Eq $($rest:tt)*) => {}; 54 (emit I32Ne $($rest:tt)*) => {}; 55 (emit I64Ne $($rest:tt)*) => {}; 56 (emit I32LtS $($rest:tt)*) => {}; 57 (emit I64LtS $($rest:tt)*) => {}; 58 (emit I32LtU $($rest:tt)*) => {}; 59 (emit I64LtU $($rest:tt)*) => {}; 60 (emit I32LeS $($rest:tt)*) => {}; 61 (emit I64LeS $($rest:tt)*) => {}; 62 (emit I32LeU $($rest:tt)*) => {}; 63 (emit I64LeU $($rest:tt)*) => {}; 64 (emit I32GtS $($rest:tt)*) => {}; 65 (emit I64GtS $($rest:tt)*) => {}; 66 (emit I32GtU $($rest:tt)*) => {}; 67 (emit I64GtU $($rest:tt)*) => {}; 68 (emit I32GeS $($rest:tt)*) => {}; 69 (emit I64GeS $($rest:tt)*) => {}; 70 (emit I32GeU $($rest:tt)*) => {}; 71 (emit I64GeU $($rest:tt)*) => {}; 72 (emit I32Eqz $($rest:tt)*) => {}; 73 (emit I64Eqz $($rest:tt)*) => {}; 74 (emit I32And $($rest:tt)*) => {}; 75 (emit I64And $($rest:tt)*) => {}; 76 (emit I32Or $($rest:tt)*) => {}; 77 (emit I64Or $($rest:tt)*) => {}; 78 (emit I32Xor $($rest:tt)*) => {}; 79 (emit I64Xor $($rest:tt)*) => {}; 80 (emit I32Shl $($rest:tt)*) => {}; 81 (emit I64Shl $($rest:tt)*) => {}; 82 (emit I32ShrS $($rest:tt)*) => {}; 83 (emit I64ShrS $($rest:tt)*) => {}; 84 (emit I32ShrU $($rest:tt)*) => {}; 85 (emit I64ShrU $($rest:tt)*) => {}; 86 (emit I32Rotl $($rest:tt)*) => {}; 87 (emit I64Rotl $($rest:tt)*) => {}; 88 (emit I32Rotr $($rest:tt)*) => {}; 89 (emit I64Rotr $($rest:tt)*) => {}; 90 (emit I32Clz $($rest:tt)*) => {}; 91 (emit I64Clz $($rest:tt)*) => {}; 92 (emit I32Ctz $($rest:tt)*) => {}; 93 (emit I64Ctz $($rest:tt)*) => {}; 94 (emit I32Popcnt $($rest:tt)*) => {}; 95 (emit I64Popcnt $($rest:tt)*) => {}; 96 (emit LocalGet $($rest:tt)*) => {}; 97 (emit LocalSet $($rest:tt)*) => {}; 98 (emit Call $($rest:tt)*) => {}; 99 (emit End $($rest:tt)*) => {}; 100 (emit Nop $($rest:tt)*) => {}; 101 (emit If $($rest:tt)*) => {}; 102 (emit Else $($rest:tt)*) => {}; 103 (emit Block $($rest:tt)*) => {}; 104 (emit Loop $($rest:tt)*) => {}; 105 (emit Br $($rest:tt)*) => {}; 106 (emit BrIf $($rest:tt)*) => {}; 107 (emit Return $($rest:tt)*) => {}; 108 (emit Unreachable $($rest:tt)*) => {}; 109 (emit LocalTee $($rest:tt)*) => {}; 110 (emit GlobalGet $($rest:tt)*) => {}; 111 (emit GlobalSet $($rest:tt)*) => {}; 112 113 (emit $unsupported:tt $($rest:tt)*) => {$($rest)*}; 114 } 115 116 impl<'a, M> VisitOperator<'a> for CodeGen<'a, M> 117 where 118 M: MacroAssembler, 119 { 120 type Output = (); 121 122 fn visit_i32_const(&mut self, val: i32) { 123 self.context.stack.push(Val::i32(val)); 124 } 125 126 fn visit_i64_const(&mut self, val: i64) { 127 self.context.stack.push(Val::i64(val)); 128 } 129 130 fn visit_i32_add(&mut self) { 131 self.context.i32_binop(self.masm, |masm, dst, src, size| { 132 masm.add(dst, dst, src, size); 133 }); 134 } 135 136 fn visit_i64_add(&mut self) { 137 self.context.i64_binop(self.masm, |masm, dst, src, size| { 138 masm.add(dst, dst, src, size); 139 }); 140 } 141 142 fn visit_i32_sub(&mut self) { 143 self.context.i32_binop(self.masm, |masm, dst, src, size| { 144 masm.sub(dst, dst, src, size); 145 }); 146 } 147 148 fn visit_i64_sub(&mut self) { 149 self.context.i64_binop(self.masm, |masm, dst, src, size| { 150 masm.sub(dst, dst, src, size); 151 }); 152 } 153 154 fn visit_i32_mul(&mut self) { 155 self.context.i32_binop(self.masm, |masm, dst, src, size| { 156 masm.mul(dst, dst, src, size); 157 }); 158 } 159 160 fn visit_i64_mul(&mut self) { 161 self.context.i64_binop(self.masm, |masm, dst, src, size| { 162 masm.mul(dst, dst, src, size); 163 }); 164 } 165 166 fn visit_i32_div_s(&mut self) { 167 use DivKind::*; 168 use OperandSize::*; 169 170 self.masm.div(&mut self.context, Signed, S32); 171 } 172 173 fn visit_i32_div_u(&mut self) { 174 use DivKind::*; 175 use OperandSize::*; 176 177 self.masm.div(&mut self.context, Unsigned, S32); 178 } 179 180 fn visit_i64_div_s(&mut self) { 181 use DivKind::*; 182 use OperandSize::*; 183 184 self.masm.div(&mut self.context, Signed, S64); 185 } 186 187 fn visit_i64_div_u(&mut self) { 188 use DivKind::*; 189 use OperandSize::*; 190 191 self.masm.div(&mut self.context, Unsigned, S64); 192 } 193 194 fn visit_i32_rem_s(&mut self) { 195 use OperandSize::*; 196 use RemKind::*; 197 198 self.masm.rem(&mut self.context, Signed, S32); 199 } 200 201 fn visit_i32_rem_u(&mut self) { 202 use OperandSize::*; 203 use RemKind::*; 204 205 self.masm.rem(&mut self.context, Unsigned, S32); 206 } 207 208 fn visit_i64_rem_s(&mut self) { 209 use OperandSize::*; 210 use RemKind::*; 211 212 self.masm.rem(&mut self.context, Signed, S64); 213 } 214 215 fn visit_i64_rem_u(&mut self) { 216 use OperandSize::*; 217 use RemKind::*; 218 219 self.masm.rem(&mut self.context, Unsigned, S64); 220 } 221 222 fn visit_i32_eq(&mut self) { 223 self.cmp_i32s(CmpKind::Eq); 224 } 225 226 fn visit_i64_eq(&mut self) { 227 self.cmp_i64s(CmpKind::Eq); 228 } 229 230 fn visit_i32_ne(&mut self) { 231 self.cmp_i32s(CmpKind::Ne); 232 } 233 234 fn visit_i64_ne(&mut self) { 235 self.cmp_i64s(CmpKind::Ne); 236 } 237 238 fn visit_i32_lt_s(&mut self) { 239 self.cmp_i32s(CmpKind::LtS); 240 } 241 242 fn visit_i64_lt_s(&mut self) { 243 self.cmp_i64s(CmpKind::LtS); 244 } 245 246 fn visit_i32_lt_u(&mut self) { 247 self.cmp_i32s(CmpKind::LtU); 248 } 249 250 fn visit_i64_lt_u(&mut self) { 251 self.cmp_i64s(CmpKind::LtU); 252 } 253 254 fn visit_i32_le_s(&mut self) { 255 self.cmp_i32s(CmpKind::LeS); 256 } 257 258 fn visit_i64_le_s(&mut self) { 259 self.cmp_i64s(CmpKind::LeS); 260 } 261 262 fn visit_i32_le_u(&mut self) { 263 self.cmp_i32s(CmpKind::LeU); 264 } 265 266 fn visit_i64_le_u(&mut self) { 267 self.cmp_i64s(CmpKind::LeU); 268 } 269 270 fn visit_i32_gt_s(&mut self) { 271 self.cmp_i32s(CmpKind::GtS); 272 } 273 274 fn visit_i64_gt_s(&mut self) { 275 self.cmp_i64s(CmpKind::GtS); 276 } 277 278 fn visit_i32_gt_u(&mut self) { 279 self.cmp_i32s(CmpKind::GtU); 280 } 281 282 fn visit_i64_gt_u(&mut self) { 283 self.cmp_i64s(CmpKind::GtU); 284 } 285 286 fn visit_i32_ge_s(&mut self) { 287 self.cmp_i32s(CmpKind::GeS); 288 } 289 290 fn visit_i64_ge_s(&mut self) { 291 self.cmp_i64s(CmpKind::GeS); 292 } 293 294 fn visit_i32_ge_u(&mut self) { 295 self.cmp_i32s(CmpKind::GeU); 296 } 297 298 fn visit_i64_ge_u(&mut self) { 299 self.cmp_i64s(CmpKind::GeU); 300 } 301 302 fn visit_i32_eqz(&mut self) { 303 use OperandSize::*; 304 305 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 306 masm.cmp_with_set(RegImm::i64(0), reg.into(), CmpKind::Eq, size); 307 }); 308 } 309 310 fn visit_i64_eqz(&mut self) { 311 use OperandSize::*; 312 313 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 314 masm.cmp_with_set(RegImm::i64(0), reg.into(), CmpKind::Eq, size); 315 }); 316 } 317 318 fn visit_i32_clz(&mut self) { 319 use OperandSize::*; 320 321 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 322 masm.clz(reg, reg, size); 323 }); 324 } 325 326 fn visit_i64_clz(&mut self) { 327 use OperandSize::*; 328 329 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 330 masm.clz(reg, reg, size); 331 }); 332 } 333 334 fn visit_i32_ctz(&mut self) { 335 use OperandSize::*; 336 337 self.context.unop(self.masm, S32, &mut |masm, reg, size| { 338 masm.ctz(reg, reg, size); 339 }); 340 } 341 342 fn visit_i64_ctz(&mut self) { 343 use OperandSize::*; 344 345 self.context.unop(self.masm, S64, &mut |masm, reg, size| { 346 masm.ctz(reg, reg, size); 347 }); 348 } 349 350 fn visit_i32_and(&mut self) { 351 self.context.i32_binop(self.masm, |masm, dst, src, size| { 352 masm.and(dst, dst, src, size); 353 }); 354 } 355 356 fn visit_i64_and(&mut self) { 357 self.context.i64_binop(self.masm, |masm, dst, src, size| { 358 masm.and(dst, dst, src, size); 359 }); 360 } 361 362 fn visit_i32_or(&mut self) { 363 self.context.i32_binop(self.masm, |masm, dst, src, size| { 364 masm.or(dst, dst, src, size); 365 }); 366 } 367 368 fn visit_i64_or(&mut self) { 369 self.context.i64_binop(self.masm, |masm, dst, src, size| { 370 masm.or(dst, dst, src, size); 371 }); 372 } 373 374 fn visit_i32_xor(&mut self) { 375 self.context.i32_binop(self.masm, |masm, dst, src, size| { 376 masm.xor(dst, dst, src, size); 377 }); 378 } 379 380 fn visit_i64_xor(&mut self) { 381 self.context.i64_binop(self.masm, |masm, dst, src, size| { 382 masm.xor(dst, dst, src, size); 383 }); 384 } 385 386 fn visit_i32_shl(&mut self) { 387 use OperandSize::*; 388 use ShiftKind::*; 389 390 self.masm.shift(&mut self.context, Shl, S32); 391 } 392 393 fn visit_i64_shl(&mut self) { 394 use OperandSize::*; 395 use ShiftKind::*; 396 397 self.masm.shift(&mut self.context, Shl, S64); 398 } 399 400 fn visit_i32_shr_s(&mut self) { 401 use OperandSize::*; 402 use ShiftKind::*; 403 404 self.masm.shift(&mut self.context, ShrS, S32); 405 } 406 407 fn visit_i64_shr_s(&mut self) { 408 use OperandSize::*; 409 use ShiftKind::*; 410 411 self.masm.shift(&mut self.context, ShrS, S64); 412 } 413 414 fn visit_i32_shr_u(&mut self) { 415 use OperandSize::*; 416 use ShiftKind::*; 417 418 self.masm.shift(&mut self.context, ShrU, S32); 419 } 420 421 fn visit_i64_shr_u(&mut self) { 422 use OperandSize::*; 423 use ShiftKind::*; 424 425 self.masm.shift(&mut self.context, ShrU, S64); 426 } 427 428 fn visit_i32_rotl(&mut self) { 429 use OperandSize::*; 430 use ShiftKind::*; 431 432 self.masm.shift(&mut self.context, Rotl, S32); 433 } 434 435 fn visit_i64_rotl(&mut self) { 436 use OperandSize::*; 437 use ShiftKind::*; 438 439 self.masm.shift(&mut self.context, Rotl, S64); 440 } 441 442 fn visit_i32_rotr(&mut self) { 443 use OperandSize::*; 444 use ShiftKind::*; 445 446 self.masm.shift(&mut self.context, Rotr, S32); 447 } 448 449 fn visit_i64_rotr(&mut self) { 450 use OperandSize::*; 451 use ShiftKind::*; 452 453 self.masm.shift(&mut self.context, Rotr, S64); 454 } 455 456 fn visit_end(&mut self) { 457 if !self.context.reachable { 458 self.handle_unreachable_end(); 459 } else { 460 let mut control = self.control_frames.pop().unwrap(); 461 let is_outermost = self.control_frames.len() == 0; 462 // If it's not the outermost control stack frame, emit the the full "end" sequence, 463 // which involves, popping results from the value stack, pushing results back to the 464 // value stack and binding the exit label. 465 // Else, pop values from the value stack and bind the exit label. 466 if !is_outermost { 467 control.emit_end(self.masm, &mut self.context); 468 } else { 469 self.context.pop_abi_results(control.result(), self.masm); 470 control.bind_exit_label(self.masm); 471 } 472 } 473 } 474 475 fn visit_i32_popcnt(&mut self) { 476 use OperandSize::*; 477 self.masm.popcnt(&mut self.context, S32); 478 } 479 480 fn visit_i64_popcnt(&mut self) { 481 use OperandSize::*; 482 483 self.masm.popcnt(&mut self.context, S64); 484 } 485 486 fn visit_local_get(&mut self, index: u32) { 487 let context = &mut self.context; 488 let slot = context 489 .frame 490 .get_local(index) 491 .expect(&format!("valid local at slot = {}", index)); 492 match slot.ty { 493 WasmType::I32 | WasmType::I64 => context.stack.push(Val::local(index)), 494 _ => panic!("Unsupported type {:?} for local", slot.ty), 495 } 496 } 497 498 // TODO verify the case where the target local is on the stack. 499 fn visit_local_set(&mut self, index: u32) { 500 let src = self.context.set_local(self.masm, index); 501 self.context.regalloc.free_gpr(src); 502 } 503 504 fn visit_call(&mut self, index: u32) { 505 self.emit_call(FuncIndex::from_u32(index)); 506 } 507 508 fn visit_nop(&mut self) {} 509 510 fn visit_if(&mut self, blockty: BlockType) { 511 self.control_frames.push(ControlStackFrame::if_( 512 &self.env.resolve_block_type(blockty), 513 self.masm, 514 &mut self.context, 515 )); 516 } 517 518 fn visit_else(&mut self) { 519 if !self.context.reachable { 520 self.handle_unreachable_else(); 521 } else { 522 let control = self 523 .control_frames 524 .last_mut() 525 .unwrap_or_else(|| panic!("Expected active control stack frame for else")); 526 control.emit_else(self.masm, &mut self.context); 527 } 528 } 529 530 fn visit_block(&mut self, blockty: BlockType) { 531 self.control_frames.push(ControlStackFrame::block( 532 &self.env.resolve_block_type(blockty), 533 self.masm, 534 &mut self.context, 535 )); 536 } 537 538 fn visit_loop(&mut self, blockty: BlockType) { 539 self.control_frames.push(ControlStackFrame::loop_( 540 &self.env.resolve_block_type(blockty), 541 self.masm, 542 &mut self.context, 543 )); 544 } 545 546 fn visit_br(&mut self, depth: u32) { 547 let frame = Self::control_at(&mut self.control_frames, depth); 548 self.context.pop_abi_results(frame.result(), self.masm); 549 self.context.pop_sp_for_branch(&frame, self.masm); 550 self.masm.jmp(*frame.label()); 551 frame.set_as_target(); 552 self.context.reachable = false; 553 } 554 555 fn visit_br_if(&mut self, depth: u32) { 556 let frame = Self::control_at(&mut self.control_frames, depth); 557 frame.set_as_target(); 558 let result = frame.result(); 559 let result_reg = self.context.gpr(result.result_reg(), self.masm); 560 let top = self.context.pop_to_reg(self.masm, None, OperandSize::S32); 561 self.context.free_gpr(result_reg); 562 self.context.pop_abi_results(result, self.masm); 563 self.context.push_abi_results(result, self.masm); 564 self.masm.branch( 565 CmpKind::Ne, 566 top.into(), 567 top.into(), 568 *frame.label(), 569 OperandSize::S32, 570 ); 571 self.context.free_gpr(top); 572 } 573 574 fn visit_return(&mut self) { 575 // Grab the outermost frame, which is the function's body frame. We 576 // don't rely on `Self::control_at` since this frame is implicit and we 577 // know that it should exist at index 0. 578 let outermost = &mut self.control_frames[0]; 579 self.context.pop_abi_results(outermost.result(), self.masm); 580 // Ensure that the stack pointer is correctly balanced. 581 self.context.pop_sp_for_branch(&outermost, self.masm); 582 // The outermost should always be a block and therefore, 583 // should always have an exit label. 584 self.masm.jmp(*outermost.exit_label().unwrap()); 585 // Set the frame as branch target so that 586 // we can bind the function's exit label. 587 outermost.set_as_target(); 588 self.context.reachable = false; 589 } 590 591 fn visit_unreachable(&mut self) { 592 self.masm.unreachable(); 593 self.context.reachable = false; 594 // Set the implicit outermost frame as target to perform the necessary 595 // stack clean up. 596 let outermost = &mut self.control_frames[0]; 597 outermost.set_as_target(); 598 } 599 600 fn visit_local_tee(&mut self, index: u32) { 601 let src = self.context.set_local(self.masm, index); 602 self.context.stack.push(Val::reg(src)); 603 } 604 605 fn visit_global_get(&mut self, global_index: u32) { 606 let index = GlobalIndex::from_u32(global_index); 607 let (ty, offset) = self.env.resolve_global_type_and_offset(index); 608 let addr = self 609 .masm 610 .address_at_reg(<M::ABI as ABI>::vmctx_reg(), offset); 611 let dst = self.context.any_gpr(self.masm); 612 self.masm.load(addr, dst, ty.into()); 613 self.context.stack.push(Val::reg(dst)); 614 } 615 616 fn visit_global_set(&mut self, global_index: u32) { 617 let index = GlobalIndex::from_u32(global_index); 618 let (ty, offset) = self.env.resolve_global_type_and_offset(index); 619 let addr = self 620 .masm 621 .address_at_reg(<M::ABI as ABI>::vmctx_reg(), offset); 622 let reg = self.context.pop_to_reg(self.masm, None, ty.into()); 623 self.context.free_gpr(reg); 624 self.masm.store(reg.into(), addr, ty.into()); 625 } 626 627 wasmparser::for_each_operator!(def_unsupported); 628 } 629 630 impl<'a, M> CodeGen<'a, M> 631 where 632 M: MacroAssembler, 633 { 634 fn cmp_i32s(&mut self, kind: CmpKind) { 635 self.context.i32_binop(self.masm, |masm, dst, src, size| { 636 masm.cmp_with_set(src, dst.get_reg().unwrap(), kind, size); 637 }); 638 } 639 640 fn cmp_i64s(&mut self, kind: CmpKind) { 641 self.context 642 .i64_binop(self.masm, move |masm, dst, src, size| { 643 masm.cmp_with_set(src, dst.get_reg().unwrap(), kind, size); 644 }); 645 } 646 } 647 648 impl From<WasmType> for OperandSize { 649 fn from(ty: WasmType) -> OperandSize { 650 match ty { 651 WasmType::I32 => OperandSize::S32, 652 WasmType::I64 => OperandSize::S64, 653 ty => todo!("unsupported type {:?}", ty), 654 } 655 } 656 } 657