1 //! Compilation support for the component model. 2 3 use crate::func_environ::BuiltinFunctions; 4 use crate::trap::TranslateTrap; 5 use crate::{TRAP_CANNOT_LEAVE_COMPONENT, TRAP_INTERNAL_ASSERT, compiler::Compiler}; 6 use cranelift_codegen::cursor::FuncCursor; 7 use cranelift_codegen::ir::condcodes::IntCC; 8 use cranelift_codegen::ir::{self, InstBuilder, MemFlags, Value}; 9 use cranelift_codegen::isa::{CallConv, TargetIsa}; 10 use cranelift_frontend::FunctionBuilder; 11 use wasmtime_environ::error::{Result, bail}; 12 use wasmtime_environ::{ 13 Abi, BuiltinFunctionIndex, CompiledFunctionBody, EntityRef, FuncKey, HostCall, PanicOnOom as _, 14 PtrSize, TrapSentinel, Tunables, WasmFuncType, WasmValType, component::*, 15 fact::PREPARE_CALL_FIXED_PARAMS, 16 }; 17 18 struct TrampolineCompiler<'a> { 19 compiler: &'a Compiler, 20 isa: &'a (dyn TargetIsa + 'static), 21 builder: FunctionBuilder<'a>, 22 component: &'a Component, 23 types: &'a ComponentTypesBuilder, 24 offsets: VMComponentOffsets<u8>, 25 block0: ir::Block, 26 signature: &'a WasmFuncType, 27 builtins: BuiltinFunctions, 28 } 29 30 /// What host functions can be called, used in `translate_hostcall` below. 31 enum HostCallee { 32 /// Call a host-lowered function specified by this index. 33 Lowering(LoweredIndex), 34 /// Call a host libcall, specified by this accessor. 35 Libcall(GetLibcallFn), 36 } 37 38 type GetLibcallFn = 39 fn(&dyn TargetIsa, &mut ir::Function) -> (ir::SigRef, ComponentBuiltinFunctionIndex); 40 41 impl From<LoweredIndex> for HostCallee { from(index: LoweredIndex) -> HostCallee42 fn from(index: LoweredIndex) -> HostCallee { 43 HostCallee::Lowering(index) 44 } 45 } 46 47 impl From<GetLibcallFn> for HostCallee { from(f: GetLibcallFn) -> HostCallee48 fn from(f: GetLibcallFn) -> HostCallee { 49 HostCallee::Libcall(f) 50 } 51 } 52 53 /// How to interpret the results of a host function. 54 enum HostResult { 55 /// The host function returns the sentinel specified which is interpreted 56 /// and translated to the real return value. 57 Sentinel(TrapSentinel), 58 59 /// The host function returns a `bool` indicating whether it succeeded or 60 /// not. 61 /// 62 /// After the return value is interpreted the host function also filled in 63 /// `ptr` and `len` with wasm return values which need to be returned. 64 /// 65 /// If `ptr` and `len` are not specified then this must be used with 66 /// `WasmArgs::ValRawList` and that ptr/len is used. 67 MultiValue { 68 /// The base pointer of the `ValRaw` list on the stack. 69 ptr: Option<ir::Value>, 70 /// The length of the `ValRaw` list on the stack. 71 len: Option<ir::Value>, 72 }, 73 } 74 75 impl From<TrapSentinel> for HostResult { from(sentinel: TrapSentinel) -> HostResult76 fn from(sentinel: TrapSentinel) -> HostResult { 77 HostResult::Sentinel(sentinel) 78 } 79 } 80 81 /// Different means of passing WebAssembly arguments to host calls. 82 #[derive(Debug, Copy, Clone)] 83 enum WasmArgs { 84 /// All wasm arguments to the host are passed directly as values, typically 85 /// through registers. 86 InRegisters, 87 88 /// All wasm arguments to the host are passed indirectly by spilling them 89 /// to the stack as a sequence of contiguous `ValRaw`s. 90 ValRawList, 91 92 /// The first `n` arguments are passed in registers, but everything after 93 /// that is spilled to the stack. 94 InRegistersUpTo(usize), 95 } 96 97 impl<'a> TrampolineCompiler<'a> { new( compiler: &'a Compiler, func_compiler: &'a mut super::FunctionCompiler<'_>, component: &'a Component, types: &'a ComponentTypesBuilder, signature: &'a WasmFuncType, ) -> TrampolineCompiler<'a>98 fn new( 99 compiler: &'a Compiler, 100 func_compiler: &'a mut super::FunctionCompiler<'_>, 101 component: &'a Component, 102 types: &'a ComponentTypesBuilder, 103 signature: &'a WasmFuncType, 104 ) -> TrampolineCompiler<'a> { 105 let isa = &*compiler.isa; 106 let func = ir::Function::with_name_signature( 107 ir::UserFuncName::user(0, 0), 108 crate::wasm_call_signature(isa, signature, &compiler.tunables), 109 ); 110 let (builder, block0) = func_compiler.builder(func); 111 TrampolineCompiler { 112 compiler, 113 isa, 114 builder, 115 component, 116 types, 117 offsets: VMComponentOffsets::new(isa.pointer_bytes(), component), 118 block0, 119 signature, 120 builtins: BuiltinFunctions::new(compiler), 121 } 122 } 123 translate(&mut self, trampoline: &Trampoline)124 fn translate(&mut self, trampoline: &Trampoline) { 125 self.check_may_leave(trampoline); 126 127 match trampoline { 128 Trampoline::Transcoder { 129 op, 130 from, 131 from64, 132 to, 133 to64, 134 } => { 135 self.translate_transcode(*op, *from, *from64, *to, *to64); 136 } 137 Trampoline::LowerImport { 138 index, 139 options, 140 lower_ty, 141 } => { 142 let pointer_type = self.isa.pointer_type(); 143 self.translate_hostcall( 144 HostCallee::Lowering(*index), 145 HostResult::MultiValue { 146 ptr: None, 147 len: None, 148 }, 149 WasmArgs::ValRawList, 150 |me, params| { 151 let vmctx = params[0]; 152 params.extend([ 153 me.builder.ins().load( 154 pointer_type, 155 MemFlags::trusted(), 156 vmctx, 157 i32::try_from(me.offsets.lowering_data(*index)).unwrap(), 158 ), 159 me.index_value(*lower_ty), 160 me.index_value(*options), 161 ]); 162 }, 163 ); 164 } 165 Trampoline::ResourceNew { instance, ty } => { 166 // Currently this only supports resources represented by `i32` 167 assert_eq!(self.signature.params()[0], WasmValType::I32); 168 self.translate_libcall( 169 host::resource_new32, 170 TrapSentinel::NegativeOne, 171 WasmArgs::InRegisters, 172 |me, params| { 173 params.push(me.index_value(*instance)); 174 params.push(me.index_value(*ty)); 175 }, 176 ); 177 } 178 Trampoline::ResourceRep { instance, ty } => { 179 // Currently this only supports resources represented by `i32` 180 assert_eq!(self.signature.results()[0], WasmValType::I32); 181 self.translate_libcall( 182 host::resource_rep32, 183 TrapSentinel::NegativeOne, 184 WasmArgs::InRegisters, 185 |me, params| { 186 params.push(me.index_value(*instance)); 187 params.push(me.index_value(*ty)); 188 }, 189 ); 190 } 191 Trampoline::ResourceDrop { instance, ty } => { 192 self.translate_resource_drop(*instance, *ty); 193 } 194 Trampoline::BackpressureInc { instance } => { 195 self.translate_libcall( 196 host::backpressure_modify, 197 TrapSentinel::Falsy, 198 WasmArgs::InRegisters, 199 |me, params| { 200 params.push(me.index_value(*instance)); 201 params.push(me.builder.ins().iconst(ir::types::I8, 1)); 202 }, 203 ); 204 } 205 Trampoline::BackpressureDec { instance } => { 206 self.translate_libcall( 207 host::backpressure_modify, 208 TrapSentinel::Falsy, 209 WasmArgs::InRegisters, 210 |me, params| { 211 params.push(me.index_value(*instance)); 212 params.push(me.builder.ins().iconst(ir::types::I8, 0)); 213 }, 214 ); 215 } 216 Trampoline::TaskReturn { 217 instance, 218 results, 219 options, 220 } => { 221 self.translate_libcall( 222 host::task_return, 223 TrapSentinel::Falsy, 224 WasmArgs::ValRawList, 225 |me, params| { 226 params.push(me.index_value(*instance)); 227 params.push(me.index_value(*results)); 228 params.push(me.index_value(*options)); 229 }, 230 ); 231 } 232 Trampoline::TaskCancel { instance } => { 233 self.translate_libcall( 234 host::task_cancel, 235 TrapSentinel::Falsy, 236 WasmArgs::InRegisters, 237 |me, params| { 238 params.push(me.index_value(*instance)); 239 }, 240 ); 241 } 242 Trampoline::WaitableSetNew { instance } => { 243 self.translate_libcall( 244 host::waitable_set_new, 245 TrapSentinel::NegativeOne, 246 WasmArgs::InRegisters, 247 |me, params| { 248 params.push(me.index_value(*instance)); 249 }, 250 ); 251 } 252 Trampoline::WaitableSetWait { instance, options } => { 253 self.translate_libcall( 254 host::waitable_set_wait, 255 TrapSentinel::NegativeOne, 256 WasmArgs::InRegisters, 257 |me, params| { 258 params.push(me.index_value(*instance)); 259 params.push(me.index_value(*options)); 260 }, 261 ); 262 } 263 Trampoline::WaitableSetPoll { instance, options } => { 264 self.translate_libcall( 265 host::waitable_set_poll, 266 TrapSentinel::NegativeOne, 267 WasmArgs::InRegisters, 268 |me, params| { 269 params.push(me.index_value(*instance)); 270 params.push(me.index_value(*options)); 271 }, 272 ); 273 } 274 Trampoline::WaitableSetDrop { instance } => { 275 self.translate_libcall( 276 host::waitable_set_drop, 277 TrapSentinel::Falsy, 278 WasmArgs::InRegisters, 279 |me, params| { 280 params.push(me.index_value(*instance)); 281 }, 282 ); 283 } 284 Trampoline::WaitableJoin { instance } => { 285 self.translate_libcall( 286 host::waitable_join, 287 TrapSentinel::Falsy, 288 WasmArgs::InRegisters, 289 |me, params| { 290 params.push(me.index_value(*instance)); 291 }, 292 ); 293 } 294 Trampoline::ThreadYield { 295 instance, 296 cancellable, 297 } => { 298 self.translate_libcall( 299 host::thread_yield, 300 TrapSentinel::NegativeOne, 301 WasmArgs::InRegisters, 302 |me, params| { 303 params.push(me.index_value(*instance)); 304 params.push( 305 me.builder 306 .ins() 307 .iconst(ir::types::I8, i64::from(*cancellable)), 308 ); 309 }, 310 ); 311 } 312 Trampoline::SubtaskDrop { instance } => { 313 self.translate_libcall( 314 host::subtask_drop, 315 TrapSentinel::Falsy, 316 WasmArgs::InRegisters, 317 |me, params| { 318 params.push(me.index_value(*instance)); 319 }, 320 ); 321 } 322 Trampoline::SubtaskCancel { instance, async_ } => { 323 self.translate_libcall( 324 host::subtask_cancel, 325 TrapSentinel::NegativeOne, 326 WasmArgs::InRegisters, 327 |me, params| { 328 params.push(me.index_value(*instance)); 329 params.push(me.builder.ins().iconst(ir::types::I8, i64::from(*async_))); 330 }, 331 ); 332 } 333 Trampoline::StreamNew { instance, ty } => { 334 self.translate_libcall( 335 host::stream_new, 336 TrapSentinel::NegativeOne, 337 WasmArgs::InRegisters, 338 |me, params| { 339 params.push(me.index_value(*instance)); 340 params.push(me.index_value(*ty)); 341 }, 342 ); 343 } 344 Trampoline::StreamRead { 345 instance, 346 ty, 347 options, 348 } => { 349 if let Some(info) = self.flat_stream_element_info(*ty).cloned() { 350 self.translate_libcall( 351 host::flat_stream_read, 352 TrapSentinel::NegativeOne, 353 WasmArgs::InRegisters, 354 |me, params| { 355 params.extend([ 356 me.index_value(*instance), 357 me.index_value(*ty), 358 me.index_value(*options), 359 me.builder 360 .ins() 361 .iconst(ir::types::I32, i64::from(info.size32)), 362 me.builder 363 .ins() 364 .iconst(ir::types::I32, i64::from(info.align32)), 365 ]); 366 }, 367 ); 368 } else { 369 self.translate_libcall( 370 host::stream_read, 371 TrapSentinel::NegativeOne, 372 WasmArgs::InRegisters, 373 |me, params| { 374 params.push(me.index_value(*instance)); 375 params.push(me.index_value(*ty)); 376 params.push(me.index_value(*options)); 377 }, 378 ); 379 } 380 } 381 Trampoline::StreamWrite { 382 instance, 383 ty, 384 options, 385 } => { 386 if let Some(info) = self.flat_stream_element_info(*ty).cloned() { 387 self.translate_libcall( 388 host::flat_stream_write, 389 TrapSentinel::NegativeOne, 390 WasmArgs::InRegisters, 391 |me, params| { 392 params.extend([ 393 me.index_value(*instance), 394 me.index_value(*ty), 395 me.index_value(*options), 396 me.builder 397 .ins() 398 .iconst(ir::types::I32, i64::from(info.size32)), 399 me.builder 400 .ins() 401 .iconst(ir::types::I32, i64::from(info.align32)), 402 ]); 403 }, 404 ); 405 } else { 406 self.translate_libcall( 407 host::stream_write, 408 TrapSentinel::NegativeOne, 409 WasmArgs::InRegisters, 410 |me, params| { 411 params.push(me.index_value(*instance)); 412 params.push(me.index_value(*ty)); 413 params.push(me.index_value(*options)); 414 }, 415 ); 416 } 417 } 418 Trampoline::StreamCancelRead { 419 instance, 420 ty, 421 async_, 422 } => { 423 self.translate_libcall( 424 host::stream_cancel_read, 425 TrapSentinel::NegativeOne, 426 WasmArgs::InRegisters, 427 |me, params| { 428 params.push(me.index_value(*instance)); 429 params.push(me.index_value(*ty)); 430 params.push(me.builder.ins().iconst(ir::types::I8, i64::from(*async_))); 431 }, 432 ); 433 } 434 Trampoline::StreamCancelWrite { 435 instance, 436 ty, 437 async_, 438 } => { 439 self.translate_libcall( 440 host::stream_cancel_write, 441 TrapSentinel::NegativeOne, 442 WasmArgs::InRegisters, 443 |me, params| { 444 params.push(me.index_value(*instance)); 445 params.push(me.index_value(*ty)); 446 params.push(me.builder.ins().iconst(ir::types::I8, i64::from(*async_))); 447 }, 448 ); 449 } 450 Trampoline::StreamDropReadable { instance, ty } => { 451 self.translate_libcall( 452 host::stream_drop_readable, 453 TrapSentinel::Falsy, 454 WasmArgs::InRegisters, 455 |me, params| { 456 params.push(me.index_value(*instance)); 457 params.push(me.index_value(*ty)); 458 }, 459 ); 460 } 461 Trampoline::StreamDropWritable { instance, ty } => { 462 self.translate_libcall( 463 host::stream_drop_writable, 464 TrapSentinel::Falsy, 465 WasmArgs::InRegisters, 466 |me, params| { 467 params.push(me.index_value(*instance)); 468 params.push(me.index_value(*ty)); 469 }, 470 ); 471 } 472 Trampoline::FutureNew { instance, ty } => { 473 self.translate_libcall( 474 host::future_new, 475 TrapSentinel::NegativeOne, 476 WasmArgs::InRegisters, 477 |me, params| { 478 params.push(me.index_value(*instance)); 479 params.push(me.index_value(*ty)); 480 }, 481 ); 482 } 483 Trampoline::FutureRead { 484 instance, 485 ty, 486 options, 487 } => { 488 self.translate_libcall( 489 host::future_read, 490 TrapSentinel::NegativeOne, 491 WasmArgs::InRegisters, 492 |me, params| { 493 params.push(me.index_value(*instance)); 494 params.push(me.index_value(*ty)); 495 params.push(me.index_value(*options)); 496 }, 497 ); 498 } 499 Trampoline::FutureWrite { 500 instance, 501 ty, 502 options, 503 } => { 504 self.translate_libcall( 505 host::future_write, 506 TrapSentinel::NegativeOne, 507 WasmArgs::InRegisters, 508 |me, params| { 509 params.push(me.index_value(*instance)); 510 params.push(me.index_value(*ty)); 511 params.push(me.index_value(*options)); 512 }, 513 ); 514 } 515 Trampoline::FutureCancelRead { 516 instance, 517 ty, 518 async_, 519 } => { 520 self.translate_libcall( 521 host::future_cancel_read, 522 TrapSentinel::NegativeOne, 523 WasmArgs::InRegisters, 524 |me, params| { 525 params.push(me.index_value(*instance)); 526 params.push(me.index_value(*ty)); 527 params.push(me.builder.ins().iconst(ir::types::I8, i64::from(*async_))); 528 }, 529 ); 530 } 531 Trampoline::FutureCancelWrite { 532 instance, 533 ty, 534 async_, 535 } => { 536 self.translate_libcall( 537 host::future_cancel_write, 538 TrapSentinel::NegativeOne, 539 WasmArgs::InRegisters, 540 |me, params| { 541 params.push(me.index_value(*instance)); 542 params.push(me.index_value(*ty)); 543 params.push(me.builder.ins().iconst(ir::types::I8, i64::from(*async_))); 544 }, 545 ); 546 } 547 Trampoline::FutureDropReadable { instance, ty } => { 548 self.translate_libcall( 549 host::future_drop_readable, 550 TrapSentinel::Falsy, 551 WasmArgs::InRegisters, 552 |me, params| { 553 params.push(me.index_value(*instance)); 554 params.push(me.index_value(*ty)); 555 }, 556 ); 557 } 558 Trampoline::FutureDropWritable { instance, ty } => { 559 self.translate_libcall( 560 host::future_drop_writable, 561 TrapSentinel::Falsy, 562 WasmArgs::InRegisters, 563 |me, params| { 564 params.push(me.index_value(*instance)); 565 params.push(me.index_value(*ty)); 566 }, 567 ); 568 } 569 Trampoline::ErrorContextNew { 570 instance, 571 ty, 572 options, 573 } => { 574 self.translate_libcall( 575 host::error_context_new, 576 TrapSentinel::NegativeOne, 577 WasmArgs::InRegisters, 578 |me, params| { 579 params.push(me.index_value(*instance)); 580 params.push(me.index_value(*ty)); 581 params.push(me.index_value(*options)); 582 }, 583 ); 584 } 585 Trampoline::ErrorContextDebugMessage { 586 instance, 587 ty, 588 options, 589 } => { 590 self.translate_libcall( 591 host::error_context_debug_message, 592 TrapSentinel::Falsy, 593 WasmArgs::InRegisters, 594 |me, params| { 595 params.push(me.index_value(*instance)); 596 params.push(me.index_value(*ty)); 597 params.push(me.index_value(*options)); 598 }, 599 ); 600 } 601 Trampoline::ErrorContextDrop { instance, ty } => { 602 self.translate_libcall( 603 host::error_context_drop, 604 TrapSentinel::Falsy, 605 WasmArgs::InRegisters, 606 |me, params| { 607 params.push(me.index_value(*instance)); 608 params.push(me.index_value(*ty)); 609 }, 610 ); 611 } 612 Trampoline::ResourceTransferOwn => { 613 self.translate_libcall( 614 host::resource_transfer_own, 615 TrapSentinel::NegativeOne, 616 WasmArgs::InRegisters, 617 |_, _| {}, 618 ); 619 } 620 Trampoline::ResourceTransferBorrow => { 621 self.translate_libcall( 622 host::resource_transfer_borrow, 623 TrapSentinel::NegativeOne, 624 WasmArgs::InRegisters, 625 |_, _| {}, 626 ); 627 } 628 Trampoline::PrepareCall { memory } => { 629 self.translate_libcall( 630 host::prepare_call, 631 TrapSentinel::Falsy, 632 WasmArgs::InRegistersUpTo(PREPARE_CALL_FIXED_PARAMS.len()), 633 |me, params| { 634 let vmctx = params[0]; 635 params.push(me.load_optional_memory(vmctx, *memory)); 636 }, 637 ); 638 } 639 Trampoline::SyncStartCall { callback } => { 640 let pointer_type = self.isa.pointer_type(); 641 let (values_vec_ptr, len) = self.compiler.allocate_stack_array_and_spill_args( 642 &WasmFuncType::new([], self.signature.results().iter().copied()).panic_on_oom(), 643 &mut self.builder, 644 &[], 645 ); 646 let values_vec_len = self.builder.ins().iconst(pointer_type, i64::from(len)); 647 self.translate_libcall( 648 host::sync_start, 649 HostResult::MultiValue { 650 ptr: Some(values_vec_ptr), 651 len: Some(values_vec_len), 652 }, 653 WasmArgs::InRegisters, 654 |me, params| { 655 let vmctx = params[0]; 656 params.push(me.load_callback(vmctx, *callback)); 657 params.push(values_vec_ptr); 658 params.push(values_vec_len); 659 }, 660 ); 661 } 662 Trampoline::AsyncStartCall { 663 callback, 664 post_return, 665 } => { 666 self.translate_libcall( 667 host::async_start, 668 TrapSentinel::NegativeOne, 669 WasmArgs::InRegisters, 670 |me, params| { 671 let vmctx = params[0]; 672 params.extend([ 673 me.load_callback(vmctx, *callback), 674 me.load_post_return(vmctx, *post_return), 675 ]); 676 }, 677 ); 678 } 679 Trampoline::FutureTransfer => { 680 self.translate_libcall( 681 host::future_transfer, 682 TrapSentinel::NegativeOne, 683 WasmArgs::InRegisters, 684 |_, _| {}, 685 ); 686 } 687 Trampoline::StreamTransfer => { 688 self.translate_libcall( 689 host::stream_transfer, 690 TrapSentinel::NegativeOne, 691 WasmArgs::InRegisters, 692 |_, _| {}, 693 ); 694 } 695 Trampoline::ErrorContextTransfer => { 696 self.translate_libcall( 697 host::error_context_transfer, 698 TrapSentinel::NegativeOne, 699 WasmArgs::InRegisters, 700 |_, _| {}, 701 ); 702 } 703 Trampoline::Trap => { 704 self.translate_libcall( 705 host::trap, 706 TrapSentinel::Falsy, 707 WasmArgs::InRegisters, 708 |_, _| {}, 709 ); 710 } 711 Trampoline::EnterSyncCall => { 712 self.translate_libcall( 713 host::enter_sync_call, 714 TrapSentinel::Falsy, 715 WasmArgs::InRegisters, 716 |_, _| {}, 717 ); 718 } 719 Trampoline::ExitSyncCall => { 720 self.translate_libcall( 721 host::exit_sync_call, 722 TrapSentinel::Falsy, 723 WasmArgs::InRegisters, 724 |_, _| {}, 725 ); 726 } 727 Trampoline::ContextGet { instance, slot } => { 728 self.translate_libcall( 729 host::context_get, 730 TrapSentinel::NegativeOne, 731 WasmArgs::InRegisters, 732 |me, params| { 733 params.push(me.index_value(*instance)); 734 params.push(me.builder.ins().iconst(ir::types::I32, i64::from(*slot))); 735 }, 736 ); 737 } 738 Trampoline::ContextSet { instance, slot } => { 739 self.translate_libcall( 740 host::context_set, 741 TrapSentinel::Falsy, 742 WasmArgs::InRegisters, 743 |me, params| { 744 params.push(me.index_value(*instance)); 745 params.push(me.builder.ins().iconst(ir::types::I32, i64::from(*slot))); 746 }, 747 ); 748 } 749 Trampoline::ThreadIndex => { 750 self.translate_libcall( 751 host::thread_index, 752 TrapSentinel::NegativeOne, 753 WasmArgs::InRegisters, 754 |_, _| {}, 755 ); 756 } 757 Trampoline::ThreadNewIndirect { 758 instance, 759 start_func_table_idx, 760 start_func_ty_idx, 761 } => { 762 self.translate_libcall( 763 host::thread_new_indirect, 764 TrapSentinel::NegativeOne, 765 WasmArgs::InRegisters, 766 |me, params| { 767 params.push(me.index_value(*instance)); 768 params.push(me.index_value(*start_func_table_idx)); 769 params.push(me.index_value(*start_func_ty_idx)); 770 }, 771 ); 772 } 773 Trampoline::ThreadSuspendToSuspended { 774 instance, 775 cancellable, 776 } => { 777 self.translate_libcall( 778 host::thread_suspend_to_suspended, 779 TrapSentinel::NegativeOne, 780 WasmArgs::InRegisters, 781 |me, params| { 782 params.push(me.index_value(*instance)); 783 params.push( 784 me.builder 785 .ins() 786 .iconst(ir::types::I8, i64::from(*cancellable)), 787 ); 788 }, 789 ); 790 } 791 Trampoline::ThreadSuspendTo { 792 instance, 793 cancellable, 794 } => { 795 self.translate_libcall( 796 host::thread_suspend_to, 797 TrapSentinel::NegativeOne, 798 WasmArgs::InRegisters, 799 |me, params| { 800 params.push(me.index_value(*instance)); 801 params.push( 802 me.builder 803 .ins() 804 .iconst(ir::types::I8, i64::from(*cancellable)), 805 ); 806 }, 807 ); 808 } 809 Trampoline::ThreadSuspend { 810 instance, 811 cancellable, 812 } => { 813 self.translate_libcall( 814 host::thread_suspend, 815 TrapSentinel::NegativeOne, 816 WasmArgs::InRegisters, 817 |me, params| { 818 params.push(me.index_value(*instance)); 819 params.push( 820 me.builder 821 .ins() 822 .iconst(ir::types::I8, i64::from(*cancellable)), 823 ); 824 }, 825 ); 826 } 827 Trampoline::ThreadUnsuspend { instance } => { 828 self.translate_libcall( 829 host::thread_unsuspend, 830 TrapSentinel::Falsy, 831 WasmArgs::InRegisters, 832 |me, params| { 833 params.push(me.index_value(*instance)); 834 }, 835 ); 836 } 837 Trampoline::ThreadYieldToSuspended { 838 instance, 839 cancellable, 840 } => { 841 self.translate_libcall( 842 host::thread_yield_to_suspended, 843 TrapSentinel::NegativeOne, 844 WasmArgs::InRegisters, 845 |me, params| { 846 params.push(me.index_value(*instance)); 847 params.push( 848 me.builder 849 .ins() 850 .iconst(ir::types::I8, i64::from(*cancellable)), 851 ); 852 }, 853 ); 854 } 855 } 856 } 857 858 /// Determine whether the specified type can be optimized as a stream 859 /// payload by lifting and lowering with a simple `memcpy`. 860 /// 861 /// Any type containing only "flat", primitive data for which all bit 862 /// patterns are valid (i.e. no pointers, handles, bools, or chars) should 863 /// qualify for this optimization, but it's also okay to conservatively 864 /// return `None` here; the fallback slow path will always work -- it just 865 /// won't be as efficient. flat_stream_element_info(&self, ty: TypeStreamTableIndex) -> Option<&CanonicalAbiInfo>866 fn flat_stream_element_info(&self, ty: TypeStreamTableIndex) -> Option<&CanonicalAbiInfo> { 867 let payload = self.types[self.types[ty].ty].payload; 868 match payload { 869 None => Some(&CanonicalAbiInfo::ZERO), 870 Some( 871 // Note that we exclude `Bool` and `Char` from this list because 872 // not all bit patterns are valid for those types. 873 payload @ (InterfaceType::S8 874 | InterfaceType::U8 875 | InterfaceType::S16 876 | InterfaceType::U16 877 | InterfaceType::S32 878 | InterfaceType::U32 879 | InterfaceType::S64 880 | InterfaceType::U64 881 | InterfaceType::Float32 882 | InterfaceType::Float64), 883 ) => Some(self.types.canonical_abi(&payload)), 884 // TODO: Recursively check for other "flat" types (i.e. those without pointers or handles), 885 // e.g. `record`s, `variant`s, etc. which contain only flat types. 886 _ => None, 887 } 888 } 889 890 /// Helper function to spill the wasm arguments `args` to this function into 891 /// a stack-allocated array. store_wasm_arguments(&mut self, args: &[Value]) -> (Value, Value)892 fn store_wasm_arguments(&mut self, args: &[Value]) -> (Value, Value) { 893 let pointer_type = self.isa.pointer_type(); 894 895 let (ptr, len) = self.compiler.allocate_stack_array_and_spill_args( 896 self.signature, 897 &mut self.builder, 898 args, 899 ); 900 let len = self.builder.ins().iconst(pointer_type, i64::from(len)); 901 (ptr, len) 902 } 903 904 /// Convenience wrapper around `translate_hostcall` to enable type inference 905 /// on the `get_libcall` parameter here. translate_libcall( &mut self, get_libcall: GetLibcallFn, host_result: impl Into<HostResult>, wasm_args: WasmArgs, extra_host_args: impl FnOnce(&mut Self, &mut Vec<ir::Value>), )906 fn translate_libcall( 907 &mut self, 908 get_libcall: GetLibcallFn, 909 host_result: impl Into<HostResult>, 910 wasm_args: WasmArgs, 911 extra_host_args: impl FnOnce(&mut Self, &mut Vec<ir::Value>), 912 ) { 913 self.translate_hostcall( 914 HostCallee::Libcall(get_libcall), 915 host_result.into(), 916 wasm_args, 917 extra_host_args, 918 ) 919 } 920 921 /// Translates an invocation of a host function and interpret the result. 922 /// 923 /// This is intended to be a relatively narrow waist which most intrinsics 924 /// go through. The configuration supported here is: 925 /// 926 /// * `host_callee` - what's being called, either a libcall or a lowered 927 /// function 928 /// * `host_result` - how to interpret the return value to see if it's a 929 /// trap 930 /// * `wasm_args` - how to pass wasm args to the host, either in registers 931 /// or on the stack 932 /// * `extra_host_args` - a closure used to push extra arguments just before 933 /// the wasm arguments are forwarded. translate_hostcall( &mut self, host_callee: HostCallee, host_result: impl Into<HostResult>, wasm_args: WasmArgs, extra_host_args: impl FnOnce(&mut Self, &mut Vec<ir::Value>), )934 fn translate_hostcall( 935 &mut self, 936 host_callee: HostCallee, 937 host_result: impl Into<HostResult>, 938 wasm_args: WasmArgs, 939 extra_host_args: impl FnOnce(&mut Self, &mut Vec<ir::Value>), 940 ) { 941 let pointer_type = self.isa.pointer_type(); 942 943 // Load all parameters in an ABI-agnostic fashion, of which the 944 // `VMComponentContext` will be the first. 945 let params = self.abi_load_params(); 946 let vmctx = params[0]; 947 let wasm_params = ¶ms[2..]; 948 949 // Start building up arguments to the host. The first is always the 950 // vmctx. After is whatever `extra_host_args` appends, and then finally 951 // is what `WasmArgs` specifies. 952 let mut host_args = vec![vmctx]; 953 extra_host_args(self, &mut host_args); 954 let mut val_raw_ptr = None; 955 let mut val_raw_len = None; 956 match wasm_args { 957 // Wasm params are passed through as values themselves. 958 WasmArgs::InRegisters => host_args.extend(wasm_params.iter().copied()), 959 960 // Wasm params are spilled and then the ptr/len is passed. 961 WasmArgs::ValRawList => { 962 let (ptr, len) = self.store_wasm_arguments(wasm_params); 963 val_raw_ptr = Some(ptr); 964 val_raw_len = Some(len); 965 host_args.push(ptr); 966 host_args.push(len); 967 } 968 969 // A mixture of the above two. 970 WasmArgs::InRegistersUpTo(n) => { 971 let (values_vec_ptr, len) = self.compiler.allocate_stack_array_and_spill_args( 972 &WasmFuncType::new(self.signature.params().iter().skip(n).copied(), []) 973 .panic_on_oom(), 974 &mut self.builder, 975 &wasm_params[n..], 976 ); 977 let values_vec_len = self.builder.ins().iconst(pointer_type, i64::from(len)); 978 979 host_args.extend(wasm_params[..n].iter().copied()); 980 host_args.push(values_vec_ptr); 981 host_args.push(values_vec_len); 982 } 983 } 984 985 // Next perform the actual invocation of the host with `host_args`. 986 let call = match host_callee { 987 HostCallee::Libcall(get_libcall) => self.call_libcall(vmctx, get_libcall, &host_args), 988 HostCallee::Lowering(index) => { 989 // Load host function pointer from the vmcontext and then call that 990 // indirect function pointer with the list of arguments. 991 let host_fn = self.builder.ins().load( 992 pointer_type, 993 MemFlags::trusted(), 994 vmctx, 995 i32::try_from(self.offsets.lowering_callee(index)).unwrap(), 996 ); 997 let host_sig = { 998 let mut sig = ir::Signature::new(CallConv::triple_default(self.isa.triple())); 999 for param in host_args.iter() { 1000 let ty = self.builder.func.dfg.value_type(*param); 1001 sig.params.push(ir::AbiParam::new(ty)); 1002 } 1003 // return value is a bool whether a trap was raised or not 1004 sig.returns.push(ir::AbiParam::new(ir::types::I8)); 1005 self.builder.import_signature(sig) 1006 }; 1007 self.compiler.call_indirect_host( 1008 &mut self.builder, 1009 HostCall::ComponentLowerImport, 1010 host_sig, 1011 host_fn, 1012 &host_args, 1013 ) 1014 } 1015 }; 1016 1017 // Acquire the result of this function (if any) and interpret it 1018 // according to `host_result`. 1019 // 1020 // Note that all match arms here end with `abi_store_results` which 1021 // accounts for the ABI of this function when storing results. 1022 let result = self.builder.func.dfg.inst_results(call).get(0).copied(); 1023 let result_ty = result.map(|v| self.builder.func.dfg.value_type(v)); 1024 let expected = self.signature.results(); 1025 match host_result.into() { 1026 HostResult::Sentinel(TrapSentinel::NegativeOne) => { 1027 assert_eq!(expected.len(), 1); 1028 let (result, result_ty) = (result.unwrap(), result_ty.unwrap()); 1029 let result = match (result_ty, expected[0]) { 1030 (ir::types::I64, WasmValType::I32) => { 1031 self.raise_if_negative_one_and_truncate(result) 1032 } 1033 (ir::types::I64, WasmValType::I64) | (ir::types::I32, WasmValType::I32) => { 1034 self.raise_if_negative_one(result) 1035 } 1036 other => panic!("unsupported NegativeOne combo {other:?}"), 1037 }; 1038 self.abi_store_results(&[result]); 1039 } 1040 HostResult::Sentinel(TrapSentinel::Falsy) => { 1041 assert_eq!(expected.len(), 0); 1042 self.raise_if_host_trapped(result.unwrap()); 1043 self.abi_store_results(&[]); 1044 } 1045 HostResult::Sentinel(_) => todo!("support additional return types if/when necessary"), 1046 1047 HostResult::MultiValue { ptr, len } => { 1048 let ptr = ptr.or(val_raw_ptr).unwrap(); 1049 let len = len.or(val_raw_len).unwrap(); 1050 self.raise_if_host_trapped(result.unwrap()); 1051 let results = self.compiler.load_values_from_array( 1052 self.signature.results(), 1053 &mut self.builder, 1054 ptr, 1055 len, 1056 ); 1057 self.abi_store_results(&results); 1058 } 1059 } 1060 } 1061 index_value(&mut self, index: impl EntityRef) -> ir::Value1062 fn index_value(&mut self, index: impl EntityRef) -> ir::Value { 1063 self.builder 1064 .ins() 1065 .iconst(ir::types::I32, i64::try_from(index.index()).unwrap()) 1066 } 1067 translate_resource_drop( &mut self, instance: RuntimeComponentInstanceIndex, resource: TypeResourceTableIndex, )1068 fn translate_resource_drop( 1069 &mut self, 1070 instance: RuntimeComponentInstanceIndex, 1071 resource: TypeResourceTableIndex, 1072 ) { 1073 let args = self.abi_load_params(); 1074 let vmctx = args[0]; 1075 let caller_vmctx = args[1]; 1076 let pointer_type = self.isa.pointer_type(); 1077 1078 // The arguments this shim passes along to the libcall are: 1079 // 1080 // * the vmctx 1081 // * the calling component instance index 1082 // * a constant value for this `ResourceDrop` intrinsic 1083 // * the wasm handle index to drop 1084 let mut host_args = Vec::new(); 1085 host_args.push(vmctx); 1086 host_args.push( 1087 self.builder 1088 .ins() 1089 .iconst(ir::types::I32, i64::from(instance.as_u32())), 1090 ); 1091 host_args.push( 1092 self.builder 1093 .ins() 1094 .iconst(ir::types::I32, i64::from(resource.as_u32())), 1095 ); 1096 host_args.push(args[2]); 1097 1098 let call = self.call_libcall(vmctx, host::resource_drop, &host_args); 1099 1100 // Immediately raise a trap if requested by the host 1101 let should_run_destructor = 1102 self.raise_if_negative_one(self.builder.func.dfg.inst_results(call)[0]); 1103 1104 let resource_ty = self.types[resource].unwrap_concrete_ty(); 1105 let resource_def = self 1106 .component 1107 .defined_resource_index(resource_ty) 1108 .map(|idx| { 1109 self.component 1110 .initializers 1111 .iter() 1112 .filter_map(|i| match i { 1113 GlobalInitializer::Resource(r) if r.index == idx => Some(r), 1114 _ => None, 1115 }) 1116 .next() 1117 .unwrap() 1118 }); 1119 let has_destructor = match resource_def { 1120 Some(def) => def.dtor.is_some(), 1121 None => true, 1122 }; 1123 // Synthesize the following: 1124 // 1125 // ... 1126 // brif should_run_destructor, run_destructor_block, return_block 1127 // 1128 // run_destructor_block: 1129 // ;; test may_leave, but only if the component instances 1130 // ;; differ 1131 // flags = load.i32 vmctx+$instance_flags_offset 1132 // masked = band flags, $FLAG_MAY_LEAVE 1133 // trapz masked, $TRAP_CANNOT_LEAVE_COMPONENT 1134 // 1135 // ;; set may_block to false, saving the old value to restore 1136 // ;; later, but only if the component instances differ and 1137 // ;; concurrency is enabled 1138 // old_may_block = load.i32 vmctx+$may_block_offset 1139 // store 0, vmctx+$may_block_offset 1140 // 1141 // ;; call enter_sync_call, but only if the component instances 1142 // ;; differ and concurrency is enabled 1143 // ... 1144 // 1145 // ;; ============================================================ 1146 // ;; this is conditionally emitted based on whether the resource 1147 // ;; has a destructor or not, and can be statically omitted 1148 // ;; because that information is known at compile time here. 1149 // rep = ushr.i64 rep, 1 1150 // rep = ireduce.i32 rep 1151 // dtor = load.ptr vmctx+$offset 1152 // func_addr = load.ptr dtor+$offset 1153 // callee_vmctx = load.ptr dtor+$offset 1154 // call_indirect func_addr, callee_vmctx, vmctx, rep 1155 // ;; ============================================================ 1156 // 1157 // ;; restore old value of may_block 1158 // store old_may_block, vmctx+$may_block_offset 1159 // 1160 // ;; if needed, call exit_sync_call 1161 // ... 1162 // 1163 // ;; if needed, restore the old value of may_block 1164 // store old_may_block, vmctx+$may_block_offset 1165 // 1166 // jump return_block 1167 // 1168 // return_block: 1169 // return 1170 // 1171 // This will decode `should_run_destructor` and run the destructor 1172 // funcref if one is specified for this resource. Note that not all 1173 // resources have destructors, hence the null check. 1174 self.builder.ensure_inserted_block(); 1175 let current_block = self.builder.current_block().unwrap(); 1176 let run_destructor_block = self.builder.create_block(); 1177 self.builder 1178 .insert_block_after(run_destructor_block, current_block); 1179 let return_block = self.builder.create_block(); 1180 self.builder 1181 .insert_block_after(return_block, run_destructor_block); 1182 1183 self.builder.ins().brif( 1184 should_run_destructor, 1185 run_destructor_block, 1186 &[], 1187 return_block, 1188 &[], 1189 ); 1190 1191 let trusted = ir::MemFlags::trusted().with_readonly(); 1192 1193 self.builder.switch_to_block(run_destructor_block); 1194 1195 // If this is a component-defined resource, the `may_leave` flag must be 1196 // checked. Additionally, if concurrency is enabled, the `may_block` 1197 // field must be updated and `enter_sync_call` called. Note though that 1198 // all of that may be elided if the resource table resides in the same 1199 // component instance that defined the resource as the component is 1200 // calling itself. 1201 let old_may_block = if let Some(def) = resource_def { 1202 if self.types[resource].unwrap_concrete_instance() != def.instance { 1203 self.check_may_leave_instance(self.types[resource].unwrap_concrete_instance()); 1204 1205 if self.compiler.tunables.concurrency_support { 1206 // Stash the old value of `may_block` and then set it to false. 1207 let old_may_block = self.builder.ins().load( 1208 ir::types::I32, 1209 trusted, 1210 vmctx, 1211 i32::try_from(self.offsets.task_may_block()).unwrap(), 1212 ); 1213 let zero = self.builder.ins().iconst(ir::types::I32, i64::from(0)); 1214 self.builder.ins().store( 1215 ir::MemFlags::trusted(), 1216 zero, 1217 vmctx, 1218 i32::try_from(self.offsets.task_may_block()).unwrap(), 1219 ); 1220 1221 // Call `enter_sync_call` 1222 // 1223 // FIXME: Apply the optimizations described in #12311. 1224 let host_args = vec![ 1225 vmctx, 1226 self.builder 1227 .ins() 1228 .iconst(ir::types::I32, i64::from(instance.as_u32())), 1229 self.builder.ins().iconst(ir::types::I32, i64::from(0)), 1230 self.builder 1231 .ins() 1232 .iconst(ir::types::I32, i64::from(def.instance.as_u32())), 1233 ]; 1234 let call = self.call_libcall(vmctx, host::enter_sync_call, &host_args); 1235 let result = self.builder.func.dfg.inst_results(call).get(0).copied(); 1236 self.raise_if_host_trapped(result.unwrap()); 1237 1238 Some(old_may_block) 1239 } else { 1240 None 1241 } 1242 } else { 1243 None 1244 } 1245 } else { 1246 None 1247 }; 1248 1249 // Conditionally emit destructor-execution code based on whether we 1250 // statically know that a destructor exists or not. 1251 if has_destructor { 1252 let rep = self.builder.ins().ushr_imm(should_run_destructor, 1); 1253 let rep = self.builder.ins().ireduce(ir::types::I32, rep); 1254 let index = self.types[resource].unwrap_concrete_ty(); 1255 // NB: despite the vmcontext storing nullable funcrefs for function 1256 // pointers we know this is statically never null due to the 1257 // `has_destructor` check above. 1258 let dtor_func_ref = self.builder.ins().load( 1259 pointer_type, 1260 trusted, 1261 vmctx, 1262 i32::try_from(self.offsets.resource_destructor(index)).unwrap(), 1263 ); 1264 if self.compiler.emit_debug_checks { 1265 self.builder 1266 .ins() 1267 .trapz(dtor_func_ref, TRAP_INTERNAL_ASSERT); 1268 } 1269 let func_addr = self.builder.ins().load( 1270 pointer_type, 1271 trusted, 1272 dtor_func_ref, 1273 i32::from(self.offsets.ptr.vm_func_ref_wasm_call()), 1274 ); 1275 let callee_vmctx = self.builder.ins().load( 1276 pointer_type, 1277 trusted, 1278 dtor_func_ref, 1279 i32::from(self.offsets.ptr.vm_func_ref_vmctx()), 1280 ); 1281 1282 let sig = crate::wasm_call_signature(self.isa, self.signature, &self.compiler.tunables); 1283 let sig_ref = self.builder.import_signature(sig); 1284 1285 // NB: note that the "caller" vmctx here is the caller of this 1286 // intrinsic itself, not the `VMComponentContext`. This effectively 1287 // takes ourselves out of the chain here but that's ok since the 1288 // caller is only used for store/limits and that same info is 1289 // stored, but elsewhere, in the component context. 1290 self.builder.ins().call_indirect( 1291 sig_ref, 1292 func_addr, 1293 &[callee_vmctx, caller_vmctx, rep], 1294 ); 1295 } 1296 1297 if let Some(old_may_block) = old_may_block { 1298 // Call `exit_sync_call` 1299 // 1300 // FIXME: Apply the optimizations described in #12311. 1301 let call = self.call_libcall(vmctx, host::exit_sync_call, &[vmctx]); 1302 let result = self.builder.func.dfg.inst_results(call).get(0).copied(); 1303 self.raise_if_host_trapped(result.unwrap()); 1304 1305 // Restore the old value of `may_block` 1306 self.builder.ins().store( 1307 ir::MemFlags::trusted(), 1308 old_may_block, 1309 vmctx, 1310 i32::try_from(self.offsets.task_may_block()).unwrap(), 1311 ); 1312 } 1313 1314 self.builder.ins().jump(return_block, &[]); 1315 self.builder.seal_block(run_destructor_block); 1316 1317 self.builder.switch_to_block(return_block); 1318 self.builder.seal_block(return_block); 1319 self.abi_store_results(&[]); 1320 } 1321 load_optional_memory( &mut self, vmctx: ir::Value, memory: Option<RuntimeMemoryIndex>, ) -> ir::Value1322 fn load_optional_memory( 1323 &mut self, 1324 vmctx: ir::Value, 1325 memory: Option<RuntimeMemoryIndex>, 1326 ) -> ir::Value { 1327 match memory { 1328 Some(idx) => self.load_memory(vmctx, idx), 1329 None => self.builder.ins().iconst(self.isa.pointer_type(), 0), 1330 } 1331 } 1332 load_memory(&mut self, vmctx: ir::Value, memory: RuntimeMemoryIndex) -> ir::Value1333 fn load_memory(&mut self, vmctx: ir::Value, memory: RuntimeMemoryIndex) -> ir::Value { 1334 self.builder.ins().load( 1335 self.isa.pointer_type(), 1336 MemFlags::trusted(), 1337 vmctx, 1338 i32::try_from(self.offsets.runtime_memory(memory)).unwrap(), 1339 ) 1340 } 1341 load_callback( &mut self, vmctx: ir::Value, callback: Option<RuntimeCallbackIndex>, ) -> ir::Value1342 fn load_callback( 1343 &mut self, 1344 vmctx: ir::Value, 1345 callback: Option<RuntimeCallbackIndex>, 1346 ) -> ir::Value { 1347 let pointer_type = self.isa.pointer_type(); 1348 match callback { 1349 Some(idx) => self.builder.ins().load( 1350 pointer_type, 1351 MemFlags::trusted(), 1352 vmctx, 1353 i32::try_from(self.offsets.runtime_callback(idx)).unwrap(), 1354 ), 1355 None => self.builder.ins().iconst(pointer_type, 0), 1356 } 1357 } 1358 load_post_return( &mut self, vmctx: ir::Value, post_return: Option<RuntimePostReturnIndex>, ) -> ir::Value1359 fn load_post_return( 1360 &mut self, 1361 vmctx: ir::Value, 1362 post_return: Option<RuntimePostReturnIndex>, 1363 ) -> ir::Value { 1364 let pointer_type = self.isa.pointer_type(); 1365 match post_return { 1366 Some(idx) => self.builder.ins().load( 1367 pointer_type, 1368 MemFlags::trusted(), 1369 vmctx, 1370 i32::try_from(self.offsets.runtime_post_return(idx)).unwrap(), 1371 ), 1372 None => self.builder.ins().iconst(pointer_type, 0), 1373 } 1374 } 1375 1376 /// Loads a host function pointer for a libcall stored at the `offset` 1377 /// provided in the libcalls array. 1378 /// 1379 /// The offset is calculated in the `host` module below. load_libcall( &mut self, vmctx: ir::Value, index: ComponentBuiltinFunctionIndex, ) -> ir::Value1380 fn load_libcall( 1381 &mut self, 1382 vmctx: ir::Value, 1383 index: ComponentBuiltinFunctionIndex, 1384 ) -> ir::Value { 1385 let pointer_type = self.isa.pointer_type(); 1386 // First load the pointer to the builtins structure which is static 1387 // per-process. 1388 let builtins_array = self.builder.ins().load( 1389 pointer_type, 1390 MemFlags::trusted().with_readonly(), 1391 vmctx, 1392 i32::try_from(self.offsets.builtins()).unwrap(), 1393 ); 1394 // Next load the function pointer at `offset` and return that. 1395 self.builder.ins().load( 1396 pointer_type, 1397 MemFlags::trusted().with_readonly(), 1398 builtins_array, 1399 i32::try_from(index.index() * u32::from(self.offsets.ptr.size())).unwrap(), 1400 ) 1401 } 1402 1403 /// Get a function's parameters regardless of the ABI in use. 1404 /// 1405 /// This emits code to load the parameters from the array-call's ABI's values 1406 /// vector, if necessary. abi_load_params(&mut self) -> Vec<ir::Value>1407 fn abi_load_params(&mut self) -> Vec<ir::Value> { 1408 self.builder.func.dfg.block_params(self.block0).to_vec() 1409 } 1410 1411 /// Emit code to return the given result values, regardless of the ABI in use. abi_store_results(&mut self, results: &[ir::Value])1412 fn abi_store_results(&mut self, results: &[ir::Value]) { 1413 self.builder.ins().return_(results); 1414 } 1415 caller_vmctx(&self) -> ir::Value1416 fn caller_vmctx(&self) -> ir::Value { 1417 self.builder.func.dfg.block_params(self.block0)[1] 1418 } 1419 raise_if_host_trapped(&mut self, succeeded: ir::Value)1420 fn raise_if_host_trapped(&mut self, succeeded: ir::Value) { 1421 let caller_vmctx = self.caller_vmctx(); 1422 self.compiler 1423 .raise_if_host_trapped(&mut self.builder, caller_vmctx, succeeded); 1424 } 1425 raise_if_transcode_trapped(&mut self, amount_copied: ir::Value)1426 fn raise_if_transcode_trapped(&mut self, amount_copied: ir::Value) { 1427 let pointer_type = self.isa.pointer_type(); 1428 let minus_one = self.builder.ins().iconst(pointer_type, -1); 1429 let succeeded = self 1430 .builder 1431 .ins() 1432 .icmp(IntCC::NotEqual, amount_copied, minus_one); 1433 self.raise_if_host_trapped(succeeded); 1434 } 1435 raise_if_negative_one_and_truncate(&mut self, ret: ir::Value) -> ir::Value1436 fn raise_if_negative_one_and_truncate(&mut self, ret: ir::Value) -> ir::Value { 1437 let ret = self.raise_if_negative_one(ret); 1438 self.builder.ins().ireduce(ir::types::I32, ret) 1439 } 1440 raise_if_negative_one(&mut self, ret: ir::Value) -> ir::Value1441 fn raise_if_negative_one(&mut self, ret: ir::Value) -> ir::Value { 1442 let result_ty = self.builder.func.dfg.value_type(ret); 1443 let minus_one = self.builder.ins().iconst(result_ty, -1); 1444 let succeeded = self.builder.ins().icmp(IntCC::NotEqual, ret, minus_one); 1445 self.raise_if_host_trapped(succeeded); 1446 ret 1447 } 1448 call_libcall( &mut self, vmctx: ir::Value, get_libcall: GetLibcallFn, args: &[ir::Value], ) -> ir::Inst1449 fn call_libcall( 1450 &mut self, 1451 vmctx: ir::Value, 1452 get_libcall: GetLibcallFn, 1453 args: &[ir::Value], 1454 ) -> ir::Inst { 1455 let (host_sig, index) = get_libcall(self.isa, &mut self.builder.func); 1456 let host_fn = self.load_libcall(vmctx, index); 1457 self.compiler 1458 .call_indirect_host(&mut self.builder, index, host_sig, host_fn, args) 1459 } 1460 check_may_leave(&mut self, trampoline: &Trampoline)1461 fn check_may_leave(&mut self, trampoline: &Trampoline) { 1462 let instance = match trampoline { 1463 // These intrinsics explicitly do not check the may-leave flag. 1464 Trampoline::ResourceRep { .. } 1465 | Trampoline::ThreadIndex 1466 | Trampoline::BackpressureInc { .. } 1467 | Trampoline::BackpressureDec { .. } 1468 | Trampoline::ContextGet { .. } 1469 | Trampoline::ContextSet { .. } => return, 1470 1471 // Intrinsics used in adapters generated by FACT that aren't called 1472 // directly from guest wasm, so no check is needed. 1473 Trampoline::ResourceTransferOwn 1474 | Trampoline::ResourceTransferBorrow 1475 | Trampoline::PrepareCall { .. } 1476 | Trampoline::SyncStartCall { .. } 1477 | Trampoline::AsyncStartCall { .. } 1478 | Trampoline::FutureTransfer 1479 | Trampoline::StreamTransfer 1480 | Trampoline::ErrorContextTransfer 1481 | Trampoline::Trap 1482 | Trampoline::EnterSyncCall 1483 | Trampoline::ExitSyncCall 1484 | Trampoline::Transcoder { .. } => return, 1485 1486 Trampoline::LowerImport { options, .. } => self.component.options[*options].instance, 1487 1488 Trampoline::ResourceNew { instance, .. } 1489 | Trampoline::ResourceDrop { instance, .. } 1490 | Trampoline::TaskReturn { instance, .. } 1491 | Trampoline::TaskCancel { instance } 1492 | Trampoline::WaitableSetNew { instance } 1493 | Trampoline::WaitableSetWait { instance, .. } 1494 | Trampoline::WaitableSetPoll { instance, .. } 1495 | Trampoline::WaitableSetDrop { instance } 1496 | Trampoline::WaitableJoin { instance } 1497 | Trampoline::ThreadYield { instance, .. } 1498 | Trampoline::ThreadNewIndirect { instance, .. } 1499 | Trampoline::ThreadSuspend { instance, .. } 1500 | Trampoline::ThreadSuspendToSuspended { instance, .. } 1501 | Trampoline::ThreadSuspendTo { instance, .. } 1502 | Trampoline::ThreadUnsuspend { instance, .. } 1503 | Trampoline::ThreadYieldToSuspended { instance, .. } 1504 | Trampoline::SubtaskDrop { instance } 1505 | Trampoline::SubtaskCancel { instance, .. } 1506 | Trampoline::ErrorContextNew { instance, .. } 1507 | Trampoline::ErrorContextDebugMessage { instance, .. } 1508 | Trampoline::ErrorContextDrop { instance, .. } 1509 | Trampoline::StreamNew { instance, .. } 1510 | Trampoline::StreamRead { instance, .. } 1511 | Trampoline::StreamWrite { instance, .. } 1512 | Trampoline::StreamCancelRead { instance, .. } 1513 | Trampoline::StreamCancelWrite { instance, .. } 1514 | Trampoline::StreamDropReadable { instance, .. } 1515 | Trampoline::StreamDropWritable { instance, .. } 1516 | Trampoline::FutureNew { instance, .. } 1517 | Trampoline::FutureRead { instance, .. } 1518 | Trampoline::FutureWrite { instance, .. } 1519 | Trampoline::FutureCancelRead { instance, .. } 1520 | Trampoline::FutureCancelWrite { instance, .. } 1521 | Trampoline::FutureDropReadable { instance, .. } 1522 | Trampoline::FutureDropWritable { instance, .. } => *instance, 1523 }; 1524 1525 self.check_may_leave_instance(instance) 1526 } 1527 check_may_leave_instance(&mut self, instance: RuntimeComponentInstanceIndex)1528 fn check_may_leave_instance(&mut self, instance: RuntimeComponentInstanceIndex) { 1529 let vmctx = self.builder.func.dfg.block_params(self.block0)[0]; 1530 1531 let flags = self.builder.ins().load( 1532 ir::types::I32, 1533 ir::MemFlags::trusted(), 1534 vmctx, 1535 i32::try_from(self.offsets.instance_flags(instance)).unwrap(), 1536 ); 1537 let may_leave_bit = self 1538 .builder 1539 .ins() 1540 .band_imm(flags, i64::from(FLAG_MAY_LEAVE)); 1541 let (mut traps, builder) = self.traps(); 1542 traps.trapz(builder, may_leave_bit, TRAP_CANNOT_LEAVE_COMPONENT); 1543 } 1544 traps(&mut self) -> (TrapTranslator<'_>, &mut FunctionBuilder<'a>)1545 fn traps(&mut self) -> (TrapTranslator<'_>, &mut FunctionBuilder<'a>) { 1546 ( 1547 TrapTranslator { 1548 compiler: self.compiler, 1549 vmctx: self.caller_vmctx(), 1550 builtins: &mut self.builtins, 1551 }, 1552 &mut self.builder, 1553 ) 1554 } 1555 } 1556 1557 // Helper structure to implement `TranslateTrap`. This isn't possible to do 1558 // natively for `TrampolineCompiler` because it stores `FunctionBuilder` 1559 // internally. This differs from `FuncEnvironment` for core wasm which stores it 1560 // externally, hence the slightly different idioms to bridge here. 1561 struct TrapTranslator<'a> { 1562 compiler: &'a Compiler, 1563 vmctx: ir::Value, 1564 builtins: &'a mut BuiltinFunctions, 1565 } 1566 1567 impl TranslateTrap for TrapTranslator<'_> { compiler(&self) -> &Compiler1568 fn compiler(&self) -> &Compiler { 1569 self.compiler 1570 } vmctx_val(&mut self, _: &mut FuncCursor<'_>) -> ir::Value1571 fn vmctx_val(&mut self, _: &mut FuncCursor<'_>) -> ir::Value { 1572 self.vmctx 1573 } builtin_funcref( &mut self, builder: &mut FunctionBuilder<'_>, index: BuiltinFunctionIndex, ) -> ir::FuncRef1574 fn builtin_funcref( 1575 &mut self, 1576 builder: &mut FunctionBuilder<'_>, 1577 index: BuiltinFunctionIndex, 1578 ) -> ir::FuncRef { 1579 self.builtins.load_builtin(builder.func, index) 1580 } 1581 } 1582 1583 impl ComponentCompiler for Compiler { compile_trampoline( &self, component: &ComponentTranslation, types: &ComponentTypesBuilder, key: FuncKey, abi: Abi, _tunables: &Tunables, symbol: &str, ) -> Result<CompiledFunctionBody>1584 fn compile_trampoline( 1585 &self, 1586 component: &ComponentTranslation, 1587 types: &ComponentTypesBuilder, 1588 key: FuncKey, 1589 abi: Abi, 1590 _tunables: &Tunables, 1591 symbol: &str, 1592 ) -> Result<CompiledFunctionBody> { 1593 let (abi2, trampoline_index) = key.unwrap_component_trampoline(); 1594 debug_assert_eq!(abi, abi2); 1595 let sig = types[component.component.trampolines[trampoline_index]].unwrap_func(); 1596 1597 match abi { 1598 // Fall through to the trampoline compiler. 1599 Abi::Wasm => {} 1600 1601 // Implement the array-abi trampoline in terms of calling the 1602 // wasm-abi trampoline. 1603 Abi::Array => { 1604 let offsets = 1605 VMComponentOffsets::new(self.isa.pointer_bytes(), &component.component); 1606 return Ok(self.array_to_wasm_trampoline( 1607 key, 1608 FuncKey::ComponentTrampoline(Abi::Wasm, trampoline_index), 1609 sig, 1610 symbol, 1611 offsets.vm_store_context(), 1612 wasmtime_environ::component::VMCOMPONENT_MAGIC, 1613 )?); 1614 } 1615 1616 Abi::Patchable => unreachable!( 1617 "We should not be compiling a patchable-ABI trampoline for a component function" 1618 ), 1619 } 1620 1621 let mut compiler = self.function_compiler(); 1622 let mut c = TrampolineCompiler::new(self, &mut compiler, &component.component, types, sig); 1623 1624 // If we are crossing the Wasm-to-native boundary, we need to save the 1625 // exit FP and return address for stack walking purposes. However, we 1626 // always debug assert that our vmctx is a component context, regardless 1627 // whether we are actually crossing that boundary because it should 1628 // always hold. 1629 let vmctx = c.builder.block_params(c.block0)[0]; 1630 let pointer_type = self.isa.pointer_type(); 1631 self.debug_assert_vmctx_kind( 1632 &mut c.builder, 1633 vmctx, 1634 wasmtime_environ::component::VMCOMPONENT_MAGIC, 1635 ); 1636 let vm_store_context = c.builder.ins().load( 1637 pointer_type, 1638 MemFlags::trusted(), 1639 vmctx, 1640 i32::try_from(c.offsets.vm_store_context()).unwrap(), 1641 ); 1642 super::save_last_wasm_exit_fp_and_pc( 1643 &mut c.builder, 1644 pointer_type, 1645 &c.offsets.ptr, 1646 vm_store_context, 1647 ); 1648 1649 c.translate(&component.trampolines[trampoline_index]); 1650 c.builder.finalize(); 1651 compiler.cx.abi = Some(abi); 1652 1653 Ok(CompiledFunctionBody { 1654 code: super::box_dyn_any_compiler_context(Some(compiler.cx)), 1655 needs_gc_heap: false, 1656 }) 1657 } 1658 compile_intrinsic( &self, _tunables: &Tunables, component: &ComponentTranslation, types: &ComponentTypesBuilder, intrinsic: UnsafeIntrinsic, abi: Abi, symbol: &str, ) -> Result<CompiledFunctionBody>1659 fn compile_intrinsic( 1660 &self, 1661 _tunables: &Tunables, 1662 component: &ComponentTranslation, 1663 types: &ComponentTypesBuilder, 1664 intrinsic: UnsafeIntrinsic, 1665 abi: Abi, 1666 symbol: &str, 1667 ) -> Result<CompiledFunctionBody> { 1668 let wasm_func_ty = WasmFuncType::new( 1669 intrinsic.core_params().iter().copied(), 1670 intrinsic.core_results().iter().copied(), 1671 ) 1672 .panic_on_oom(); 1673 1674 match abi { 1675 // Fall through to the trampoline compiler. 1676 Abi::Wasm => {} 1677 1678 // Implement the array-abi trampoline in terms of calling the 1679 // wasm-abi trampoline. 1680 Abi::Array => { 1681 let offsets = 1682 VMComponentOffsets::new(self.isa.pointer_bytes(), &component.component); 1683 return Ok(self.array_to_wasm_trampoline( 1684 FuncKey::UnsafeIntrinsic(abi, intrinsic), 1685 FuncKey::UnsafeIntrinsic(Abi::Wasm, intrinsic), 1686 &wasm_func_ty, 1687 symbol, 1688 offsets.vm_store_context(), 1689 wasmtime_environ::component::VMCOMPONENT_MAGIC, 1690 )?); 1691 } 1692 1693 Abi::Patchable => { 1694 unreachable!( 1695 "We should not be compiling a patchable trampoline for a component intrinsic" 1696 ) 1697 } 1698 } 1699 1700 let mut compiler = self.function_compiler(); 1701 let mut c = TrampolineCompiler::new( 1702 self, 1703 &mut compiler, 1704 &component.component, 1705 &types, 1706 &wasm_func_ty, 1707 ); 1708 1709 match intrinsic { 1710 UnsafeIntrinsic::U8NativeLoad 1711 | UnsafeIntrinsic::U16NativeLoad 1712 | UnsafeIntrinsic::U32NativeLoad 1713 | UnsafeIntrinsic::U64NativeLoad => c.translate_load_intrinsic(intrinsic)?, 1714 UnsafeIntrinsic::U8NativeStore 1715 | UnsafeIntrinsic::U16NativeStore 1716 | UnsafeIntrinsic::U32NativeStore 1717 | UnsafeIntrinsic::U64NativeStore => c.translate_store_intrinsic(intrinsic)?, 1718 UnsafeIntrinsic::StoreDataAddress => { 1719 let [callee_vmctx, _caller_vmctx] = *c.abi_load_params() else { 1720 unreachable!() 1721 }; 1722 let pointer_type = self.isa.pointer_type(); 1723 1724 // Load the `*mut VMStoreContext` out of our vmctx. 1725 let store_ctx = c.builder.ins().load( 1726 pointer_type, 1727 ir::MemFlags::trusted() 1728 .with_readonly() 1729 .with_alias_region(Some(ir::AliasRegion::Vmctx)) 1730 .with_can_move(), 1731 callee_vmctx, 1732 i32::try_from(c.offsets.vm_store_context()).unwrap(), 1733 ); 1734 1735 // Load the `*mut T` out of the `VMStoreContext`. 1736 let data_address = c.builder.ins().load( 1737 pointer_type, 1738 ir::MemFlags::trusted() 1739 .with_readonly() 1740 .with_alias_region(Some(ir::AliasRegion::Vmctx)) 1741 .with_can_move(), 1742 store_ctx, 1743 i32::from(c.offsets.ptr.vmstore_context_store_data()), 1744 ); 1745 1746 // Zero-extend the address if we are on a 32-bit architecture. 1747 let data_address = match pointer_type.bits() { 1748 32 => c.builder.ins().uextend(ir::types::I64, data_address), 1749 64 => data_address, 1750 p => bail!("unsupported architecture: no support for {p}-bit pointers"), 1751 }; 1752 1753 c.abi_store_results(&[data_address]); 1754 } 1755 } 1756 1757 c.builder.finalize(); 1758 compiler.cx.abi = Some(abi); 1759 1760 Ok(CompiledFunctionBody { 1761 code: super::box_dyn_any_compiler_context(Some(compiler.cx)), 1762 needs_gc_heap: false, 1763 }) 1764 } 1765 } 1766 1767 macro_rules! unsafe_intrinsic_clif_params_results { 1768 ( 1769 $( 1770 $symbol:expr => $variant:ident : $ctor:ident ( $( $param:ident : $param_ty:ident ),* ) $( -> $result_ty:ident )? ; 1771 )* 1772 ) => { 1773 fn unsafe_intrinsic_clif_params(intrinsic: UnsafeIntrinsic) -> &'static [ir::types::Type] { 1774 match intrinsic { 1775 $( 1776 UnsafeIntrinsic::$variant => &[ $( unsafe_intrinsic_clif_params_results!(@clif_type $param_ty) ),* ], 1777 )* 1778 } 1779 } 1780 1781 fn unsafe_intrinsic_clif_results(intrinsic: UnsafeIntrinsic) -> &'static [ir::types::Type] { 1782 match intrinsic { 1783 $( 1784 UnsafeIntrinsic::$variant => &[ $( unsafe_intrinsic_clif_params_results!(@clif_type $result_ty) )? ], 1785 )* 1786 } 1787 } 1788 }; 1789 1790 (@clif_type u8) => { ir::types::I8 }; 1791 (@clif_type u16) => { ir::types::I16 }; 1792 (@clif_type u32) => { ir::types::I32 }; 1793 (@clif_type u64) => { ir::types::I64 }; 1794 } 1795 1796 wasmtime_environ::for_each_unsafe_intrinsic!(unsafe_intrinsic_clif_params_results); 1797 1798 impl TrampolineCompiler<'_> { translate_transcode( &mut self, op: Transcode, from: RuntimeMemoryIndex, from64: bool, to: RuntimeMemoryIndex, to64: bool, )1799 fn translate_transcode( 1800 &mut self, 1801 op: Transcode, 1802 from: RuntimeMemoryIndex, 1803 from64: bool, 1804 to: RuntimeMemoryIndex, 1805 to64: bool, 1806 ) { 1807 let pointer_type = self.isa.pointer_type(); 1808 let vmctx = self.builder.func.dfg.block_params(self.block0)[0]; 1809 1810 // Determine the static signature of the host libcall for this transcode 1811 // operation and additionally calculate the static offset within the 1812 // transode libcalls array. 1813 let get_libcall = match op { 1814 Transcode::Copy(FixedEncoding::Utf8) => host::utf8_to_utf8, 1815 Transcode::Copy(FixedEncoding::Utf16) => host::utf16_to_utf16, 1816 Transcode::Copy(FixedEncoding::Latin1) => host::latin1_to_latin1, 1817 Transcode::Latin1ToUtf16 => host::latin1_to_utf16, 1818 Transcode::Latin1ToUtf8 => host::latin1_to_utf8, 1819 Transcode::Utf16ToCompactProbablyUtf16 => host::utf16_to_compact_probably_utf16, 1820 Transcode::Utf16ToCompactUtf16 => host::utf16_to_compact_utf16, 1821 Transcode::Utf16ToLatin1 => host::utf16_to_latin1, 1822 Transcode::Utf16ToUtf8 => host::utf16_to_utf8, 1823 Transcode::Utf8ToCompactUtf16 => host::utf8_to_compact_utf16, 1824 Transcode::Utf8ToLatin1 => host::utf8_to_latin1, 1825 Transcode::Utf8ToUtf16 => host::utf8_to_utf16, 1826 }; 1827 1828 // Load the base pointers for the from/to linear memories. 1829 let from_base = self.load_runtime_memory_base(vmctx, from); 1830 let to_base = self.load_runtime_memory_base(vmctx, to); 1831 1832 let mut args = Vec::new(); 1833 args.push(vmctx); 1834 1835 let uses_retptr = match op { 1836 Transcode::Utf16ToUtf8 1837 | Transcode::Latin1ToUtf8 1838 | Transcode::Utf8ToLatin1 1839 | Transcode::Utf16ToLatin1 => true, 1840 _ => false, 1841 }; 1842 1843 // Most transcoders share roughly the same signature despite doing very 1844 // different things internally, so most libcalls are lumped together 1845 // here. 1846 match op { 1847 Transcode::Copy(_) 1848 | Transcode::Latin1ToUtf16 1849 | Transcode::Utf16ToCompactProbablyUtf16 1850 | Transcode::Utf8ToLatin1 1851 | Transcode::Utf16ToLatin1 1852 | Transcode::Utf8ToUtf16 => { 1853 args.push(self.ptr_param(0, from64, from_base)); 1854 args.push(self.len_param(1, from64)); 1855 args.push(self.ptr_param(2, to64, to_base)); 1856 } 1857 1858 Transcode::Utf16ToUtf8 | Transcode::Latin1ToUtf8 => { 1859 args.push(self.ptr_param(0, from64, from_base)); 1860 args.push(self.len_param(1, from64)); 1861 args.push(self.ptr_param(2, to64, to_base)); 1862 args.push(self.len_param(3, to64)); 1863 } 1864 1865 Transcode::Utf8ToCompactUtf16 | Transcode::Utf16ToCompactUtf16 => { 1866 args.push(self.ptr_param(0, from64, from_base)); 1867 args.push(self.len_param(1, from64)); 1868 args.push(self.ptr_param(2, to64, to_base)); 1869 args.push(self.len_param(3, to64)); 1870 args.push(self.len_param(4, to64)); 1871 } 1872 }; 1873 if uses_retptr { 1874 let slot = self 1875 .builder 1876 .func 1877 .create_sized_stack_slot(ir::StackSlotData::new( 1878 ir::StackSlotKind::ExplicitSlot, 1879 pointer_type.bytes(), 1880 0, 1881 )); 1882 args.push(self.builder.ins().stack_addr(pointer_type, slot, 0)); 1883 } 1884 let call = self.call_libcall(vmctx, get_libcall, &args); 1885 let mut results = self.builder.func.dfg.inst_results(call).to_vec(); 1886 if uses_retptr { 1887 results.push(self.builder.ins().load( 1888 pointer_type, 1889 ir::MemFlags::trusted(), 1890 *args.last().unwrap(), 1891 0, 1892 )); 1893 } 1894 let mut raw_results = Vec::new(); 1895 1896 // Like the arguments the results are fairly similar across libcalls, so 1897 // they're lumped into various buckets here. 1898 match op { 1899 Transcode::Copy(_) | Transcode::Latin1ToUtf16 => { 1900 self.raise_if_host_trapped(results[0]); 1901 } 1902 1903 Transcode::Utf8ToUtf16 1904 | Transcode::Utf16ToCompactProbablyUtf16 1905 | Transcode::Utf8ToCompactUtf16 1906 | Transcode::Utf16ToCompactUtf16 => { 1907 self.raise_if_transcode_trapped(results[0]); 1908 raw_results.push(self.cast_from_pointer(results[0], to64)); 1909 } 1910 1911 Transcode::Latin1ToUtf8 1912 | Transcode::Utf16ToUtf8 1913 | Transcode::Utf8ToLatin1 1914 | Transcode::Utf16ToLatin1 => { 1915 self.raise_if_transcode_trapped(results[0]); 1916 raw_results.push(self.cast_from_pointer(results[0], from64)); 1917 raw_results.push(self.cast_from_pointer(results[1], to64)); 1918 } 1919 }; 1920 1921 self.builder.ins().return_(&raw_results); 1922 } 1923 1924 // Helper function to cast an input parameter to the host pointer type. len_param(&mut self, param: usize, is64: bool) -> ir::Value1925 fn len_param(&mut self, param: usize, is64: bool) -> ir::Value { 1926 let val = self.builder.func.dfg.block_params(self.block0)[2 + param]; 1927 self.cast_to_pointer(val, is64) 1928 } 1929 1930 // Helper function to interpret an input parameter as a pointer into 1931 // linear memory. This will cast the input parameter to the host integer 1932 // type and then add that value to the base. 1933 // 1934 // Note that bounds-checking happens in adapter modules, and this 1935 // trampoline is simply calling the host libcall. ptr_param(&mut self, param: usize, is64: bool, base: ir::Value) -> ir::Value1936 fn ptr_param(&mut self, param: usize, is64: bool, base: ir::Value) -> ir::Value { 1937 let val = self.len_param(param, is64); 1938 self.builder.ins().iadd(base, val) 1939 } 1940 1941 // Helper function to cast a core wasm input to a host pointer type 1942 // which will go into the host libcall. cast_to_pointer(&mut self, val: ir::Value, is64: bool) -> ir::Value1943 fn cast_to_pointer(&mut self, val: ir::Value, is64: bool) -> ir::Value { 1944 let pointer_type = self.isa.pointer_type(); 1945 let host64 = pointer_type == ir::types::I64; 1946 if is64 == host64 { 1947 val 1948 } else if !is64 { 1949 assert!(host64); 1950 self.builder.ins().uextend(pointer_type, val) 1951 } else { 1952 assert!(!host64); 1953 self.builder.ins().ireduce(pointer_type, val) 1954 } 1955 } 1956 1957 // Helper to cast a host pointer integer type to the destination type. cast_from_pointer(&mut self, val: ir::Value, is64: bool) -> ir::Value1958 fn cast_from_pointer(&mut self, val: ir::Value, is64: bool) -> ir::Value { 1959 let host64 = self.isa.pointer_type() == ir::types::I64; 1960 if is64 == host64 { 1961 val 1962 } else if !is64 { 1963 assert!(host64); 1964 self.builder.ins().ireduce(ir::types::I32, val) 1965 } else { 1966 assert!(!host64); 1967 self.builder.ins().uextend(ir::types::I64, val) 1968 } 1969 } 1970 load_runtime_memory_base(&mut self, vmctx: ir::Value, mem: RuntimeMemoryIndex) -> ir::Value1971 fn load_runtime_memory_base(&mut self, vmctx: ir::Value, mem: RuntimeMemoryIndex) -> ir::Value { 1972 let pointer_type = self.isa.pointer_type(); 1973 let from_vmmemory_definition = self.load_memory(vmctx, mem); 1974 self.builder.ins().load( 1975 pointer_type, 1976 MemFlags::trusted(), 1977 from_vmmemory_definition, 1978 i32::from(self.offsets.ptr.vmmemory_definition_base()), 1979 ) 1980 } 1981 translate_load_intrinsic(&mut self, intrinsic: UnsafeIntrinsic) -> Result<()>1982 fn translate_load_intrinsic(&mut self, intrinsic: UnsafeIntrinsic) -> Result<()> { 1983 // Emit code for a native-load intrinsic. 1984 debug_assert_eq!(intrinsic.core_params(), &[WasmValType::I64]); 1985 debug_assert_eq!(intrinsic.core_results().len(), 1); 1986 1987 let wasm_ty = intrinsic.core_results()[0]; 1988 let clif_ty = unsafe_intrinsic_clif_results(intrinsic)[0]; 1989 1990 let [_callee_vmctx, _caller_vmctx, pointer] = *self.abi_load_params() else { 1991 unreachable!() 1992 }; 1993 1994 // Truncate the pointer, if necessary. 1995 debug_assert_eq!(self.builder.func.dfg.value_type(pointer), ir::types::I64); 1996 let pointer = match self.isa.pointer_bits() { 1997 32 => self.builder.ins().ireduce(ir::types::I32, pointer), 1998 64 => pointer, 1999 p => bail!("unsupported architecture: no support for {p}-bit pointers"), 2000 }; 2001 2002 // Do the load! 2003 let mut value = self 2004 .builder 2005 .ins() 2006 .load(clif_ty, ir::MemFlags::trusted(), pointer, 0); 2007 2008 // Extend the value, if necessary. When implementing the 2009 // `u8-native-load` intrinsic, for example, we will load a Cranelift 2010 // value of type `i8` but we need to extend it to an `i32` because 2011 // Wasm doesn't have an `i8` core value type. 2012 let wasm_clif_ty = crate::value_type(self.isa, wasm_ty); 2013 if clif_ty != wasm_clif_ty { 2014 assert!(clif_ty.bytes() < wasm_clif_ty.bytes()); 2015 // NB: all of our unsafe intrinsics for native loads are 2016 // unsigned, so we always zero-extend. 2017 value = self.builder.ins().uextend(wasm_clif_ty, value); 2018 } 2019 2020 self.abi_store_results(&[value]); 2021 Ok(()) 2022 } 2023 translate_store_intrinsic(&mut self, intrinsic: UnsafeIntrinsic) -> Result<()>2024 fn translate_store_intrinsic(&mut self, intrinsic: UnsafeIntrinsic) -> Result<()> { 2025 debug_assert!(intrinsic.core_results().is_empty()); 2026 debug_assert!(matches!(intrinsic.core_params(), [WasmValType::I64, _])); 2027 2028 let wasm_ty = intrinsic.core_params()[1]; 2029 let clif_ty = unsafe_intrinsic_clif_params(intrinsic)[1]; 2030 2031 let [_callee_vmctx, _caller_vmctx, pointer, mut value] = *self.abi_load_params() else { 2032 unreachable!() 2033 }; 2034 2035 // Truncate the pointer, if necessary. 2036 debug_assert_eq!(self.builder.func.dfg.value_type(pointer), ir::types::I64); 2037 let pointer = match self.isa.pointer_bits() { 2038 32 => self.builder.ins().ireduce(ir::types::I32, pointer), 2039 64 => pointer, 2040 p => bail!("unsupported architecture: no support for {p}-bit pointers"), 2041 }; 2042 2043 // Truncate the value, if necessary. For example, with 2044 // `u8-native-store` we will be given an `i32` from Wasm (because 2045 // core Wasm does not have an 8-bit integer value type) and we need 2046 // to reduce that into an `i8`. 2047 let wasm_ty = crate::value_type(self.isa, wasm_ty); 2048 if clif_ty != wasm_ty { 2049 assert!(clif_ty.bytes() < wasm_ty.bytes()); 2050 value = self.builder.ins().ireduce(clif_ty, value); 2051 } 2052 2053 // Do the store! 2054 self.builder 2055 .ins() 2056 .store(ir::MemFlags::trusted(), value, pointer, 0); 2057 2058 self.abi_store_results(&[]); 2059 Ok(()) 2060 } 2061 } 2062 2063 /// Module with macro-generated contents that will return the signature and 2064 /// offset for each of the host transcoder functions. 2065 /// 2066 /// Note that a macro is used here to keep this in sync with the actual 2067 /// transcoder functions themselves which are also defined via a macro. 2068 mod host { 2069 use cranelift_codegen::ir::{self, AbiParam}; 2070 use cranelift_codegen::isa::{CallConv, TargetIsa}; 2071 use wasmtime_environ::component::ComponentBuiltinFunctionIndex; 2072 2073 macro_rules! define { 2074 ( 2075 $( 2076 $( #[$attr:meta] )* 2077 $name:ident( $( $pname:ident: $param:ident ),* ) $( -> $result:ident )?; 2078 )* 2079 ) => { 2080 $( 2081 pub(super) fn $name(isa: &dyn TargetIsa, func: &mut ir::Function) -> (ir::SigRef, ComponentBuiltinFunctionIndex) { 2082 let pointer_type = isa.pointer_type(); 2083 let sig = build_sig( 2084 isa, 2085 func, 2086 &[$( define!(@ty pointer_type $param) ),*], 2087 &[$( define!(@ty pointer_type $result) ),*], 2088 ); 2089 2090 return (sig, ComponentBuiltinFunctionIndex::$name()) 2091 } 2092 )* 2093 }; 2094 2095 (@ty $ptr:ident size) => ($ptr); 2096 (@ty $ptr:ident ptr_u8) => ($ptr); 2097 (@ty $ptr:ident ptr_u16) => ($ptr); 2098 (@ty $ptr:ident ptr_size) => ($ptr); 2099 (@ty $ptr:ident bool) => (ir::types::I8); 2100 (@ty $ptr:ident u8) => (ir::types::I8); 2101 (@ty $ptr:ident u32) => (ir::types::I32); 2102 (@ty $ptr:ident u64) => (ir::types::I64); 2103 (@ty $ptr:ident vmctx) => ($ptr); 2104 } 2105 2106 wasmtime_environ::foreach_builtin_component_function!(define); 2107 build_sig( isa: &dyn TargetIsa, func: &mut ir::Function, params: &[ir::Type], returns: &[ir::Type], ) -> ir::SigRef2108 fn build_sig( 2109 isa: &dyn TargetIsa, 2110 func: &mut ir::Function, 2111 params: &[ir::Type], 2112 returns: &[ir::Type], 2113 ) -> ir::SigRef { 2114 let mut sig = ir::Signature { 2115 params: params.iter().map(|ty| AbiParam::new(*ty)).collect(), 2116 returns: returns.iter().map(|ty| AbiParam::new(*ty)).collect(), 2117 call_conv: CallConv::triple_default(isa.triple()), 2118 }; 2119 2120 // Once we're declaring the signature of a host function we must respect 2121 // the default ABI of the platform which is where argument extension of 2122 // params/results may come into play. 2123 let extension = isa.default_argument_extension(); 2124 for arg in sig.params.iter_mut().chain(sig.returns.iter_mut()) { 2125 if arg.value_type.is_int() { 2126 arg.extension = extension; 2127 } 2128 } 2129 func.import_signature(sig) 2130 } 2131 } 2132