1 use crate::{
2 Result,
3 abi::{ABIOperand, ABISig, RetArea, vmctx},
4 bail,
5 codegen::BlockSig,
6 ensure, format_err,
7 isa::reg::{Reg, RegClass, writable},
8 masm::{
9 AtomicWaitKind, Extend, Imm, IntCmpKind, IntScratch, LaneSelector, LoadKind,
10 MacroAssembler, OperandSize, RegImm, RmwOp, SPOffset, ShiftKind, StoreKind, TrapCode,
11 UNTRUSTED_FLAGS, Zero,
12 },
13 stack::{TypedReg, Val},
14 };
15 use cranelift_codegen::{
16 binemit::CodeOffset,
17 ir::{RelSourceLoc, SourceLoc},
18 };
19 use smallvec::SmallVec;
20 use std::marker::PhantomData;
21 use wasmparser::{
22 BinaryReader, FuncValidator, MemArg, Operator, OperatorsReader, ValidatorResources,
23 VisitOperator, VisitSimdOperator,
24 };
25 use wasmtime_cranelift::{TRAP_BAD_SIGNATURE, TRAP_HEAP_MISALIGNED, TRAP_TABLE_OUT_OF_BOUNDS};
26 use wasmtime_environ::{
27 FUNCREF_MASK, GlobalIndex, MemoryIndex, PtrSize, TableIndex, Tunables, TypeIndex, WasmHeapType,
28 WasmValType,
29 };
30
31 mod context;
32 pub(crate) use context::*;
33 mod env;
34 pub use env::*;
35 mod call;
36 pub(crate) use call::*;
37 mod control;
38 pub(crate) use control::*;
39 mod builtin;
40 pub use builtin::*;
41 pub(crate) mod bounds;
42
43 use bounds::{Bounds, ImmOffset, Index};
44
45 mod phase;
46 pub(crate) use phase::*;
47
48 mod error;
49 pub(crate) use error::*;
50
51 /// Branch states in the compiler, enabling the derivation of the
52 /// reachability state.
53 pub(crate) trait BranchState {
54 /// Whether the compiler will enter in an unreachable state after
55 /// the branch is emitted.
unreachable_state_after_emission() -> bool56 fn unreachable_state_after_emission() -> bool;
57 }
58
59 /// A conditional branch state, with a fallthrough.
60 pub(crate) struct ConditionalBranch;
61
62 impl BranchState for ConditionalBranch {
unreachable_state_after_emission() -> bool63 fn unreachable_state_after_emission() -> bool {
64 false
65 }
66 }
67
68 /// Unconditional branch state.
69 pub(crate) struct UnconditionalBranch;
70
71 impl BranchState for UnconditionalBranch {
unreachable_state_after_emission() -> bool72 fn unreachable_state_after_emission() -> bool {
73 true
74 }
75 }
76
77 /// Holds metadata about the source code location and the machine code emission.
78 /// The fields of this struct are opaque and are not interpreted in any way.
79 /// They serve as a mapping between source code and machine code.
80 #[derive(Default)]
81 pub(crate) struct SourceLocation {
82 /// The base source location.
83 pub base: Option<SourceLoc>,
84 /// The current relative source code location along with its associated
85 /// machine code offset.
86 pub current: (CodeOffset, RelSourceLoc),
87 }
88
89 /// The code generation abstraction.
90 pub(crate) struct CodeGen<'a, 'translation: 'a, 'data: 'translation, M, P>
91 where
92 M: MacroAssembler,
93 P: CodeGenPhase,
94 {
95 /// The ABI-specific representation of the function signature, excluding results.
96 pub sig: ABISig,
97
98 /// The code generation context.
99 pub context: CodeGenContext<'a, P>,
100
101 /// A reference to the function compilation environment.
102 pub env: FuncEnv<'a, 'translation, 'data, M::Ptr>,
103
104 /// The MacroAssembler.
105 pub masm: &'a mut M,
106
107 /// Stack frames for control flow.
108 // NB The 64 is set arbitrarily, we can adjust it as
109 // we see fit.
110 pub control_frames: SmallVec<[ControlStackFrame; 64]>,
111
112 /// Information about the source code location.
113 pub source_location: SourceLocation,
114
115 /// Compilation settings for code generation.
116 pub tunables: &'a Tunables,
117
118 /// Local counter to track fuel consumption.
119 pub fuel_consumed: i64,
120 phase: PhantomData<P>,
121 }
122
123 impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M, Prologue>
124 where
125 M: MacroAssembler,
126 {
new( tunables: &'a Tunables, masm: &'a mut M, context: CodeGenContext<'a, Prologue>, env: FuncEnv<'a, 'translation, 'data, M::Ptr>, sig: ABISig, ) -> CodeGen<'a, 'translation, 'data, M, Prologue>127 pub fn new(
128 tunables: &'a Tunables,
129 masm: &'a mut M,
130 context: CodeGenContext<'a, Prologue>,
131 env: FuncEnv<'a, 'translation, 'data, M::Ptr>,
132 sig: ABISig,
133 ) -> CodeGen<'a, 'translation, 'data, M, Prologue> {
134 Self {
135 sig,
136 context,
137 masm,
138 env,
139 tunables,
140 source_location: Default::default(),
141 control_frames: Default::default(),
142 // Empty functions should consume at least 1 fuel unit.
143 fuel_consumed: 1,
144 phase: PhantomData,
145 }
146 }
147
148 /// Code generation prologue.
emit_prologue(mut self) -> Result<CodeGen<'a, 'translation, 'data, M, Emission>>149 pub fn emit_prologue(mut self) -> Result<CodeGen<'a, 'translation, 'data, M, Emission>> {
150 let vmctx = self
151 .sig
152 .params()
153 .first()
154 .ok_or_else(|| format_err!(CodeGenError::vmcontext_arg_expected()))?
155 .unwrap_reg();
156
157 self.masm.start_source_loc(Default::default())?;
158 // We need to use the vmctx parameter before pinning it for stack checking.
159 self.masm.prologue(vmctx)?;
160
161 // Pin the `VMContext` pointer.
162 self.masm.mov(
163 writable!(vmctx!(M)),
164 vmctx.into(),
165 self.env.ptr_type().try_into()?,
166 )?;
167
168 self.masm.reserve_stack(self.context.frame.locals_size)?;
169 self.spill_register_arguments()?;
170
171 let defined_locals_range = &self.context.frame.defined_locals_range;
172 self.masm.zero_mem_range(defined_locals_range.as_range())?;
173
174 // Save the results base parameter register into its slot.
175
176 if self.sig.params.has_retptr() {
177 match self.sig.params.unwrap_results_area_operand() {
178 ABIOperand::Reg { ty, reg, .. } => {
179 let results_base_slot = self.context.frame.results_base_slot.as_ref().unwrap();
180 ensure!(
181 results_base_slot.addressed_from_sp(),
182 CodeGenError::sp_addressing_expected(),
183 );
184 let addr = self.masm.local_address(results_base_slot)?;
185 self.masm.store((*reg).into(), addr, (*ty).try_into()?)?;
186 }
187 // The result base parameter is a stack parameter, addressed
188 // from FP.
189 _ => {}
190 }
191 }
192
193 self.masm.end_source_loc()?;
194
195 Ok(CodeGen {
196 sig: self.sig,
197 context: self.context.for_emission(),
198 masm: self.masm,
199 env: self.env,
200 tunables: self.tunables,
201 source_location: self.source_location,
202 control_frames: self.control_frames,
203 fuel_consumed: self.fuel_consumed,
204 phase: PhantomData,
205 })
206 }
207
spill_register_arguments(&mut self) -> Result<()>208 fn spill_register_arguments(&mut self) -> Result<()> {
209 use WasmValType::*;
210 for (operand, slot) in self
211 .sig
212 .params_without_retptr()
213 .iter()
214 .zip(self.context.frame.locals())
215 {
216 match (operand, slot) {
217 (ABIOperand::Reg { ty, reg, .. }, slot) => {
218 let addr = self.masm.local_address(slot)?;
219 match &ty {
220 I32 | I64 | F32 | F64 | V128 => {
221 self.masm.store((*reg).into(), addr, (*ty).try_into()?)?;
222 }
223 Ref(rt) => match rt.heap_type {
224 WasmHeapType::Func | WasmHeapType::Extern => {
225 self.masm.store_ptr(*reg, addr)?;
226 }
227 _ => bail!(CodeGenError::unsupported_wasm_type()),
228 },
229 }
230 }
231 // Skip non-register arguments
232 _ => {}
233 }
234 }
235 Ok(())
236 }
237 }
238
239 impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M, Emission>
240 where
241 M: MacroAssembler,
242 {
243 /// Emit the function body to machine code.
emit( &mut self, body: BinaryReader<'a>, validator: &mut FuncValidator<ValidatorResources>, ) -> Result<()>244 pub fn emit(
245 &mut self,
246 body: BinaryReader<'a>,
247 validator: &mut FuncValidator<ValidatorResources>,
248 ) -> Result<()> {
249 self.emit_body(body, validator)
250 .and_then(|_| self.emit_end())?;
251
252 Ok(())
253 }
254
255 /// Pops a control frame from the control frame stack.
pop_control_frame(&mut self) -> Result<ControlStackFrame>256 pub fn pop_control_frame(&mut self) -> Result<ControlStackFrame> {
257 self.control_frames
258 .pop()
259 .ok_or_else(|| format_err!(CodeGenError::control_frame_expected()))
260 }
261
262 /// Derives a [RelSourceLoc] from a [SourceLoc].
source_loc_from(&mut self, loc: SourceLoc) -> RelSourceLoc263 pub fn source_loc_from(&mut self, loc: SourceLoc) -> RelSourceLoc {
264 if self.source_location.base.is_none() && !loc.is_default() {
265 self.source_location.base = Some(loc);
266 }
267
268 RelSourceLoc::from_base_offset(self.source_location.base.unwrap_or_default(), loc)
269 }
270
271 /// The following two helpers, handle else or end instructions when the
272 /// compiler has entered into an unreachable code state. These instructions
273 /// must be observed to determine if the reachability state should be
274 /// restored.
275 ///
276 /// When the compiler is in an unreachable state, all the other instructions
277 /// are not visited.
handle_unreachable_else(&mut self) -> Result<()>278 pub fn handle_unreachable_else(&mut self) -> Result<()> {
279 let frame = self
280 .control_frames
281 .last_mut()
282 .ok_or_else(|| CodeGenError::control_frame_expected())?;
283 ensure!(frame.is_if(), CodeGenError::if_control_frame_expected());
284 if frame.is_next_sequence_reachable() {
285 // We entered an unreachable state when compiling the
286 // if-then branch, but if the `if` was reachable at
287 // entry, the if-else branch will be reachable.
288 self.context.reachable = true;
289 frame.ensure_stack_state(self.masm, &mut self.context)?;
290 frame.bind_else(self.masm, &mut self.context)?;
291 }
292 Ok(())
293 }
294
handle_unreachable_end(&mut self) -> Result<()>295 pub fn handle_unreachable_end(&mut self) -> Result<()> {
296 let mut frame = self.pop_control_frame()?;
297 // We just popped the outermost block.
298 let is_outermost = self.control_frames.len() == 0;
299
300 if frame.is_next_sequence_reachable() {
301 self.context.reachable = true;
302 frame.ensure_stack_state(self.masm, &mut self.context)?;
303 frame.bind_end(self.masm, &mut self.context)
304 } else if is_outermost {
305 // If we reach the end of the function in an unreachable
306 // state, perform the necessary cleanup to leave the stack
307 // and SP in the expected state. The compiler can enter
308 // in this state through an infinite loop.
309 frame.ensure_stack_state(self.masm, &mut self.context)
310 } else {
311 Ok(())
312 }
313 }
314
emit_body( &mut self, body: BinaryReader<'a>, validator: &mut FuncValidator<ValidatorResources>, ) -> Result<()>315 fn emit_body(
316 &mut self,
317 body: BinaryReader<'a>,
318 validator: &mut FuncValidator<ValidatorResources>,
319 ) -> Result<()> {
320 self.maybe_emit_fuel_check()?;
321
322 self.maybe_emit_epoch_check()?;
323
324 // Once we have emitted the epilogue and reserved stack space for the locals, we push the
325 // base control flow block.
326 self.control_frames.push(ControlStackFrame::block(
327 BlockSig::from_sig(self.sig.clone()),
328 self.masm,
329 &mut self.context,
330 )?);
331
332 // Set the return area of the results *after* initializing the block. In
333 // the function body block case, we'll treat the results as any other
334 // case, addressed from the stack pointer, and when ending the function
335 // the return area will be set to the return pointer.
336 if self.sig.params.has_retptr() {
337 self.sig
338 .results
339 .set_ret_area(RetArea::slot(self.context.frame.results_base_slot.unwrap()));
340 }
341
342 let mut ops = OperatorsReader::new(body);
343 while !ops.eof() {
344 let offset = ops.original_position();
345 ops.visit_operator(&mut ValidateThenVisit(
346 validator.simd_visitor(offset),
347 self,
348 offset,
349 ))??;
350 }
351 ops.finish()?;
352 return Ok(());
353
354 struct ValidateThenVisit<'a, T, U>(T, &'a mut U, usize);
355
356 macro_rules! validate_then_visit {
357 ($( @$proposal:ident $op:ident $({ $($arg:ident: $argty:ty),* })? => $visit:ident $ann:tt)*) => {
358 $(
359 fn $visit(&mut self $($(,$arg: $argty)*)?) -> Self::Output {
360 self.0.$visit($($($arg.clone()),*)?)?;
361 let op = Operator::$op $({ $($arg: $arg.clone()),* })?;
362 if self.1.visit(&op) {
363 self.1.before_visit_op(&op, self.2)?;
364 let res = self.1.$visit($($($arg),*)?)?;
365 self.1.after_visit_op()?;
366 Ok(res)
367 } else {
368 Ok(())
369 }
370 }
371 )*
372 };
373 }
374
375 fn visit_op_when_unreachable(op: &Operator) -> bool {
376 use Operator::*;
377 match op {
378 If { .. } | Block { .. } | Loop { .. } | Else | End => true,
379 _ => false,
380 }
381 }
382
383 /// Trait to handle hooks that must happen before and after visiting an
384 /// operator.
385 trait VisitorHooks {
386 /// Hook prior to visiting an operator.
387 fn before_visit_op(&mut self, operator: &Operator, offset: usize) -> Result<()>;
388 /// Hook after visiting an operator.
389 fn after_visit_op(&mut self) -> Result<()>;
390
391 /// Returns `true` if the operator will be visited.
392 ///
393 /// Operators will be visited if the following invariants are met:
394 /// * The compiler is in a reachable state.
395 /// * The compiler is in an unreachable state, but the current
396 /// operator is a control flow operator. These operators need to be
397 /// visited in order to keep the control stack frames balanced and
398 /// to determine if the reachability state must be restored.
399 fn visit(&self, op: &Operator) -> bool;
400 }
401
402 impl<'a, 'translation, 'data, M: MacroAssembler> VisitorHooks
403 for CodeGen<'a, 'translation, 'data, M, Emission>
404 {
405 fn visit(&self, op: &Operator) -> bool {
406 self.context.reachable || visit_op_when_unreachable(op)
407 }
408
409 fn before_visit_op(&mut self, operator: &Operator, offset: usize) -> Result<()> {
410 // Handle source location mapping.
411 self.source_location_before_visit_op(offset)?;
412
413 // Handle fuel.
414 if self.tunables.consume_fuel {
415 self.fuel_before_visit_op(operator)?;
416 }
417 Ok(())
418 }
419
420 fn after_visit_op(&mut self) -> Result<()> {
421 // Handle source code location mapping.
422 self.source_location_after_visit_op()
423 }
424 }
425
426 impl<'a, T, U> VisitOperator<'a> for ValidateThenVisit<'_, T, U>
427 where
428 T: VisitSimdOperator<'a, Output = wasmparser::Result<()>>,
429 U: VisitSimdOperator<'a, Output = Result<()>> + VisitorHooks,
430 {
431 type Output = U::Output;
432
433 fn simd_visitor(
434 &mut self,
435 ) -> Option<&mut dyn VisitSimdOperator<'a, Output = Self::Output>>
436 where
437 T:,
438 {
439 Some(self)
440 }
441
442 wasmparser::for_each_visit_operator!(validate_then_visit);
443 }
444
445 impl<'a, T, U> VisitSimdOperator<'a> for ValidateThenVisit<'_, T, U>
446 where
447 T: VisitSimdOperator<'a, Output = wasmparser::Result<()>>,
448 U: VisitSimdOperator<'a, Output = Result<()>> + VisitorHooks,
449 {
450 wasmparser::for_each_visit_simd_operator!(validate_then_visit);
451 }
452 }
453
454 /// Emits a a series of instructions that will type check a function reference call.
emit_typecheck_funcref( &mut self, funcref_ptr: Reg, type_index: TypeIndex, ) -> Result<()>455 pub fn emit_typecheck_funcref(
456 &mut self,
457 funcref_ptr: Reg,
458 type_index: TypeIndex,
459 ) -> Result<()> {
460 let ptr_size: OperandSize = self.env.ptr_type().try_into()?;
461 let sig_index_bytes = self.env.vmoffsets.size_of_vmshared_type_index();
462 let sig_size = OperandSize::from_bytes(sig_index_bytes);
463 let sig_index = self.env.translation.module.types[type_index].unwrap_module_type_index();
464 let sig_offset = sig_index
465 .as_u32()
466 .checked_mul(sig_index_bytes.into())
467 .unwrap();
468 let signatures_base_offset = self.env.vmoffsets.ptr.vmctx_type_ids_array();
469 let funcref_sig_offset = self.env.vmoffsets.ptr.vm_func_ref_type_index();
470 // Get the caller id.
471 let caller_id = self.context.any_gpr(self.masm)?;
472
473 self.masm.with_scratch::<IntScratch, _>(|masm, scratch| {
474 // Load the signatures address into the scratch register.
475 masm.load(
476 masm.address_at_vmctx(signatures_base_offset.into())?,
477 scratch.writable(),
478 ptr_size,
479 )?;
480
481 masm.load(
482 masm.address_at_reg(scratch.inner(), sig_offset)?,
483 writable!(caller_id),
484 sig_size,
485 )
486 })?;
487
488 let callee_id = self.context.any_gpr(self.masm)?;
489 self.masm.load(
490 self.masm
491 .address_at_reg(funcref_ptr, funcref_sig_offset.into())?,
492 writable!(callee_id),
493 sig_size,
494 )?;
495
496 // Typecheck.
497 self.masm
498 .cmp(caller_id, callee_id.into(), OperandSize::S32)?;
499 self.masm.trapif(IntCmpKind::Ne, TRAP_BAD_SIGNATURE)?;
500 self.context.free_reg(callee_id);
501 self.context.free_reg(caller_id);
502 wasmtime_environ::error::Ok(())
503 }
504
505 /// Emit the usual function end instruction sequence.
emit_end(&mut self) -> Result<()>506 fn emit_end(&mut self) -> Result<()> {
507 // The implicit body block is treated a normal block (it pushes results
508 // to the stack); so when reaching the end, we pop them taking as
509 // reference the current function's signature.
510 let base = SPOffset::from_u32(self.context.frame.locals_size);
511 self.masm.start_source_loc(Default::default())?;
512 if self.context.reachable {
513 ControlStackFrame::pop_abi_results_impl(
514 &mut self.sig.results,
515 &mut self.context,
516 self.masm,
517 |results, _, _| Ok(results.ret_area().copied()),
518 )?;
519 } else {
520 // If we reach the end of the function in an unreachable code state,
521 // simply truncate to the expected values.
522 // The compiler could enter this state through an infinite loop.
523 self.context.truncate_stack_to(0)?;
524 self.masm.reset_stack_pointer(base)?;
525 }
526 ensure!(
527 self.context.stack.len() == 0,
528 CodeGenError::unexpected_value_in_value_stack()
529 );
530 self.masm.free_stack(self.context.frame.locals_size)?;
531 self.masm.epilogue()?;
532 self.masm.end_source_loc()?;
533 Ok(())
534 }
535
536 /// Pops the value at the stack top and assigns it to the local at
537 /// the given index, returning the typed register holding the
538 /// source value.
emit_set_local(&mut self, index: u32) -> Result<TypedReg>539 pub fn emit_set_local(&mut self, index: u32) -> Result<TypedReg> {
540 // Materialize any references to the same local index that are in the
541 // value stack by spilling.
542 if self.context.stack.contains_latent_local(index) {
543 self.context.spill(self.masm)?;
544 }
545 let src = self.context.pop_to_reg(self.masm, None)?;
546 // Need to get address of local after `pop_to_reg` since `pop_to_reg`
547 // will pop the machine stack causing an incorrect address to be
548 // calculated.
549 let (ty, addr) = self.context.frame.get_local_address(index, self.masm)?;
550 self.masm
551 .store(RegImm::reg(src.reg), addr, ty.try_into()?)?;
552
553 Ok(src)
554 }
555
556 /// Loads the address of the given global.
emit_get_global_addr(&mut self, index: GlobalIndex) -> Result<(WasmValType, Reg, u32)>557 pub fn emit_get_global_addr(&mut self, index: GlobalIndex) -> Result<(WasmValType, Reg, u32)> {
558 let data = self.env.resolve_global(index);
559
560 if data.imported {
561 let global_base = self.masm.address_at_reg(vmctx!(M), data.offset)?;
562 let dst = self.context.any_gpr(self.masm)?;
563 self.masm.load_ptr(global_base, writable!(dst))?;
564 Ok((data.ty, dst, 0))
565 } else {
566 Ok((data.ty, vmctx!(M), data.offset))
567 }
568 }
569
emit_lazy_init_funcref(&mut self, table_index: TableIndex) -> Result<()>570 pub fn emit_lazy_init_funcref(&mut self, table_index: TableIndex) -> Result<()> {
571 assert!(self.tunables.table_lazy_init, "unsupported eager init");
572 let table_data = self.env.resolve_table_data(table_index);
573 let ptr_type = self.env.ptr_type();
574 let builtin = self.env.builtins.table_get_lazy_init_func_ref::<M::ABI>()?;
575
576 // Request the builtin's result register and use it to hold the table
577 // element value. We preemptively spill and request this register to
578 // avoid conflict at the control flow merge below. Requesting the result
579 // register is safe since we know ahead-of-time the builtin's signature.
580 self.context.spill(self.masm)?;
581 let elem_value: Reg = self.context.reg(
582 builtin.sig().results.unwrap_singleton().unwrap_reg(),
583 self.masm,
584 )?;
585
586 let index = self.context.pop_to_reg(self.masm, None)?;
587 let base = self.context.any_gpr(self.masm)?;
588
589 let elem_addr = self.emit_compute_table_elem_addr(index.into(), base, &table_data)?;
590 self.masm.load_ptr(elem_addr, writable!(elem_value))?;
591 // Free the register used as base, once we have loaded the element
592 // address into the element value register.
593 self.context.free_reg(base);
594
595 let (defined, cont) = (self.masm.get_label()?, self.masm.get_label()?);
596
597 // Push the built-in arguments to the stack.
598 self.context
599 .stack
600 .extend([table_index.as_u32().try_into().unwrap(), index.into()]);
601
602 self.masm.branch(
603 IntCmpKind::Ne,
604 elem_value,
605 elem_value.into(),
606 defined,
607 ptr_type.try_into()?,
608 )?;
609 // Free the element value register.
610 // This is safe since the FnCall::emit call below, will ensure
611 // that the result register is placed on the value stack.
612 self.context.free_reg(elem_value);
613 FnCall::emit::<M>(
614 &mut self.env,
615 self.masm,
616 &mut self.context,
617 Callee::Builtin(builtin.clone()),
618 )?;
619
620 // We know the signature of the libcall in this case, so we assert that there's
621 // one element in the stack and that it's the ABI signature's result register.
622 let top = self
623 .context
624 .stack
625 .peek()
626 .ok_or_else(|| CodeGenError::missing_values_in_stack())?;
627 let top = top.unwrap_reg();
628 ensure!(
629 top.reg == elem_value,
630 CodeGenError::table_element_value_expected()
631 );
632 self.masm.jmp(cont)?;
633
634 // In the defined case, mask the funcref address in place, by peeking into the
635 // last element of the value stack, which was pushed by the `indirect` function
636 // call above.
637 //
638 // Note that `FUNCREF_MASK` as type `usize` but here we want a 64-bit
639 // value so assert its actual value and then use a `-2` literal.
640 self.masm.bind(defined)?;
641 assert_eq!(FUNCREF_MASK as isize, -2);
642 let imm = RegImm::i64(-2);
643 let dst = top.into();
644 self.masm
645 .and(writable!(dst), dst, imm, top.ty.try_into()?)?;
646
647 self.masm.bind(cont)
648 }
649
650 /// Emits a series of instructions to bounds check and calculate the address
651 /// of the given WebAssembly memory.
652 /// This function returns a register containing the requested address.
653 ///
654 /// In essence, when computing the heap address for a WebAssembly load or
655 /// store instruction the objective is to ensure that such access is safe,
656 /// but also to perform the least amount of checks, and rely on the system to
657 /// detect illegal memory accesses where applicable.
658 ///
659 /// Winch follows almost the same principles as Cranelift when it comes to
660 /// bounds checks, for a more detailed explanation refer to
661 /// prepare_addr in wasmtime-cranelift.
662 ///
663 /// Winch implementation differs in that, it defaults to the general case
664 /// for dynamic heaps rather than optimizing for doing the least amount of
665 /// work possible at runtime, this is done to align with Winch's principle
666 /// of doing the least amount of work possible at compile time. For static
667 /// heaps, Winch does a bit more of work, given that some of the cases that
668 /// are checked against, can benefit compilation times, like for example,
669 /// detecting an out of bounds access at compile time.
emit_compute_heap_address( &mut self, memarg: &MemArg, access_size: OperandSize, ) -> Result<Option<Reg>>670 pub fn emit_compute_heap_address(
671 &mut self,
672 memarg: &MemArg,
673 access_size: OperandSize,
674 ) -> Result<Option<Reg>> {
675 let ptr_size: OperandSize = self.env.ptr_type().try_into()?;
676 let enable_spectre_mitigation = self.env.heap_access_spectre_mitigation();
677 let add_offset_and_access_size = |offset: ImmOffset, access_size: OperandSize| {
678 (access_size.bytes() as u64) + (offset.as_u32() as u64)
679 };
680
681 let memory_index = MemoryIndex::from_u32(memarg.memory);
682 let heap = self.env.resolve_heap(memory_index);
683 let index = Index::from_typed_reg(self.context.pop_to_reg(self.masm, None)?);
684
685 let offset = bounds::ensure_index_and_offset(
686 self.masm,
687 index,
688 memarg.offset,
689 heap.index_type().try_into()?,
690 )?;
691 let offset_with_access_size = add_offset_and_access_size(offset, access_size);
692
693 let can_elide_bounds_check = heap
694 .memory
695 .can_elide_bounds_check(self.tunables, self.env.page_size_log2);
696
697 let addr = if offset_with_access_size > heap.memory.maximum_byte_size().unwrap_or(u64::MAX)
698 || (!self.tunables.memory_may_move
699 && offset_with_access_size > self.tunables.memory_reservation)
700 {
701 // Detect at compile time if the access is out of bounds.
702 // Doing so will put the compiler in an unreachable code state,
703 // optimizing the work that the compiler has to do until the
704 // reachability is restored or when reaching the end of the
705 // function.
706
707 self.emit_fuel_increment()?;
708 self.masm.trap(TrapCode::HEAP_OUT_OF_BOUNDS)?;
709 self.context.reachable = false;
710 None
711
712 // Account for the case in which we can completely elide the bounds
713 // checks.
714 //
715 // This case, makes use of the fact that if a memory access uses
716 // a 32-bit index, then we be certain that
717 //
718 // index <= u32::MAX
719 //
720 // Therefore if any 32-bit index access occurs in the region
721 // represented by
722 //
723 // bound + guard_size - (offset + access_size)
724 //
725 // We are certain that it's in bounds or that the underlying virtual
726 // memory subsystem will report an illegal access at runtime.
727 //
728 // Note:
729 //
730 // * bound - (offset + access_size) cannot wrap, because it's checked
731 // in the condition above.
732 // * bound + heap.offset_guard_size is guaranteed to not overflow if
733 // the heap configuration is correct, given that it's address must
734 // fit in 64-bits.
735 // * If the heap type is 32-bits, the offset is at most u32::MAX, so
736 // no adjustment is needed as part of
737 // [bounds::ensure_index_and_offset].
738 } else if can_elide_bounds_check
739 && u64::from(u32::MAX)
740 <= self.tunables.memory_reservation + self.tunables.memory_guard_size
741 - offset_with_access_size
742 {
743 assert!(can_elide_bounds_check);
744 assert!(heap.index_type() == WasmValType::I32);
745 let addr = self.context.any_gpr(self.masm)?;
746 bounds::load_heap_addr_unchecked(self.masm, &heap, index, offset, addr, ptr_size)?;
747 Some(addr)
748
749 // Account for the case of a static memory size. The access is out
750 // of bounds if:
751 //
752 // index > bound - (offset + access_size)
753 //
754 // bound - (offset + access_size) cannot wrap, because we already
755 // checked that (offset + access_size) > bound, above.
756 } else if let Some(static_size) = heap.memory.static_heap_size() {
757 let bounds = Bounds::from_u64(static_size);
758 let addr = bounds::load_heap_addr_checked(
759 self.masm,
760 &mut self.context,
761 ptr_size,
762 &heap,
763 enable_spectre_mitigation,
764 bounds,
765 index,
766 offset,
767 |masm, bounds, index| {
768 let adjusted_bounds = bounds.as_u64() - offset_with_access_size;
769 let index_reg = index.as_typed_reg().reg;
770 masm.cmp(
771 index_reg,
772 RegImm::i64(adjusted_bounds as i64),
773 // Similar to the dynamic heap case, even though the
774 // offset and access size are bound through the heap
775 // type, when added they can overflow, resulting in
776 // an erroneous comparison, therefore we rely on the
777 // target pointer size.
778 ptr_size,
779 )?;
780 Ok(IntCmpKind::GtU)
781 },
782 )?;
783 Some(addr)
784 } else {
785 // Account for the general case for bounds-checked memories. The
786 // access is out of bounds if:
787 // * index + offset + access_size overflows
788 // OR
789 // * index + offset + access_size > bound
790 let bounds = bounds::load_dynamic_heap_bounds::<_>(
791 &mut self.context,
792 self.masm,
793 &heap,
794 ptr_size,
795 )?;
796
797 let index_reg = index.as_typed_reg().reg;
798 // Allocate a temporary register to hold
799 // index + offset + access_size
800 // which will serve as the check condition.
801 let index_offset_and_access_size = self.context.any_gpr(self.masm)?;
802
803 // Move the value of the index to the
804 // index_offset_and_access_size register to perform the overflow
805 // check to avoid clobbering the initial index value.
806 //
807 // We derive size of the operation from the heap type since:
808 //
809 // * This is the first assignment to the
810 // `index_offset_and_access_size` register
811 //
812 // * The memory64 proposal specifies that the index is bound to
813 // the heap type instead of hardcoding it to 32-bits (i32).
814 self.masm.mov(
815 writable!(index_offset_and_access_size),
816 index_reg.into(),
817 heap.index_type().try_into()?,
818 )?;
819 // Perform
820 // index = index + offset + access_size, trapping if the
821 // addition overflows.
822 //
823 // We use the target's pointer size rather than depending on the heap
824 // type since we want to check for overflow; even though the
825 // offset and access size are guaranteed to be bounded by the heap
826 // type, when added, if used with the wrong operand size, their
827 // result could be clamped, resulting in an erroneous overflow
828 // check.
829 self.masm.checked_uadd(
830 writable!(index_offset_and_access_size),
831 index_offset_and_access_size,
832 Imm::i64(offset_with_access_size as i64),
833 ptr_size,
834 TrapCode::HEAP_OUT_OF_BOUNDS,
835 )?;
836
837 let addr = bounds::load_heap_addr_checked(
838 self.masm,
839 &mut self.context,
840 ptr_size,
841 &heap,
842 enable_spectre_mitigation,
843 bounds,
844 index,
845 offset,
846 |masm, bounds, _| {
847 let bounds_reg = bounds.as_typed_reg().reg;
848 masm.cmp(
849 index_offset_and_access_size,
850 bounds_reg.into(),
851 // We use the pointer size to keep the bounds
852 // comparison consistent with the result of the
853 // overflow check above.
854 ptr_size,
855 )?;
856 Ok(IntCmpKind::GtU)
857 },
858 )?;
859 self.context.free_reg(bounds.as_typed_reg().reg);
860 self.context.free_reg(index_offset_and_access_size);
861 Some(addr)
862 };
863
864 self.context.free_reg(index.as_typed_reg().reg);
865 Ok(addr)
866 }
867
868 /// Emit checks to ensure that the address at `memarg` is correctly aligned for `size`.
emit_check_align(&mut self, memarg: &MemArg, size: OperandSize) -> Result<()>869 fn emit_check_align(&mut self, memarg: &MemArg, size: OperandSize) -> Result<()> {
870 if size.bytes() > 1 {
871 // Peek addr from top of the stack by popping and pushing.
872 let addr = *self
873 .context
874 .stack
875 .peek()
876 .ok_or_else(|| CodeGenError::missing_values_in_stack())?;
877 let tmp = self.context.any_gpr(self.masm)?;
878 self.context.move_val_to_reg(&addr, tmp, self.masm)?;
879
880 if memarg.offset != 0 {
881 self.masm.add(
882 writable!(tmp),
883 tmp,
884 RegImm::Imm(Imm::I64(memarg.offset)),
885 size,
886 )?;
887 }
888
889 self.masm.and(
890 writable!(tmp),
891 tmp,
892 RegImm::Imm(Imm::I32(size.bytes() - 1)),
893 size,
894 )?;
895
896 self.masm.cmp(tmp, RegImm::Imm(Imm::i64(0)), size)?;
897 self.masm.trapif(IntCmpKind::Ne, TRAP_HEAP_MISALIGNED)?;
898 self.context.free_reg(tmp);
899 }
900
901 Ok(())
902 }
903
emit_compute_heap_address_align_checked( &mut self, memarg: &MemArg, access_size: OperandSize, ) -> Result<Option<Reg>>904 pub fn emit_compute_heap_address_align_checked(
905 &mut self,
906 memarg: &MemArg,
907 access_size: OperandSize,
908 ) -> Result<Option<Reg>> {
909 self.emit_check_align(memarg, access_size)?;
910 self.emit_compute_heap_address(memarg, access_size)
911 }
912
913 /// Emit a WebAssembly load.
emit_wasm_load( &mut self, arg: &MemArg, target_type: WasmValType, kind: LoadKind, ) -> Result<()>914 pub fn emit_wasm_load(
915 &mut self,
916 arg: &MemArg,
917 target_type: WasmValType,
918 kind: LoadKind,
919 ) -> Result<()> {
920 let emit_load = |this: &mut Self, dst, addr, kind| -> Result<()> {
921 let src = this.masm.address_at_reg(addr, 0)?;
922 this.masm.wasm_load(src, writable!(dst), kind)?;
923 this.context
924 .stack
925 .push(TypedReg::new(target_type, dst).into());
926 this.context.free_reg(addr);
927 Ok(())
928 };
929
930 // Ensure that the destination register is not allocated if
931 // `emit_compute_heap_address` does not return an address.
932 match kind {
933 LoadKind::VectorLane(_) => {
934 // Destination vector register is at the top of the stack and
935 // `emit_compute_heap_address` expects an integer register
936 // containing the address to load to be at the top of the stack.
937 let dst = self.context.pop_to_reg(self.masm, None)?;
938 let addr = self.emit_compute_heap_address(&arg, kind.derive_operand_size())?;
939 if let Some(addr) = addr {
940 emit_load(self, dst.reg, addr, kind)?;
941 } else {
942 self.context.free_reg(dst);
943 }
944 }
945 _ => {
946 let maybe_addr = match kind {
947 LoadKind::Atomic(_, _) => self.emit_compute_heap_address_align_checked(
948 &arg,
949 kind.derive_operand_size(),
950 )?,
951 _ => self.emit_compute_heap_address(&arg, kind.derive_operand_size())?,
952 };
953
954 if let Some(addr) = maybe_addr {
955 let dst = match target_type {
956 WasmValType::I32 | WasmValType::I64 => self.context.any_gpr(self.masm)?,
957 WasmValType::F32 | WasmValType::F64 => self.context.any_fpr(self.masm)?,
958 WasmValType::V128 => self.context.reg_for_type(target_type, self.masm)?,
959 _ => bail!(CodeGenError::unsupported_wasm_type()),
960 };
961
962 emit_load(self, dst, addr, kind)?;
963 }
964 }
965 }
966
967 Ok(())
968 }
969
970 /// Emit a WebAssembly store.
emit_wasm_store(&mut self, arg: &MemArg, kind: StoreKind) -> Result<()>971 pub fn emit_wasm_store(&mut self, arg: &MemArg, kind: StoreKind) -> Result<()> {
972 let src = self.context.pop_to_reg(self.masm, None)?;
973
974 let maybe_addr = match kind {
975 StoreKind::Atomic(size) => self.emit_compute_heap_address_align_checked(&arg, size)?,
976 StoreKind::Operand(size) | StoreKind::VectorLane(LaneSelector { size, .. }) => {
977 self.emit_compute_heap_address(&arg, size)?
978 }
979 };
980
981 if let Some(addr) = maybe_addr {
982 self.masm
983 .wasm_store(src.reg, self.masm.address_at_reg(addr, 0)?, kind)?;
984
985 self.context.free_reg(addr);
986 }
987 self.context.free_reg(src);
988
989 Ok(())
990 }
991
992 /// Loads the address of the table element at a given index. Returns the
993 /// address of the table element using the provided register as base.
emit_compute_table_elem_addr( &mut self, index: Reg, base: Reg, table_data: &TableData, ) -> Result<M::Address>994 pub fn emit_compute_table_elem_addr(
995 &mut self,
996 index: Reg,
997 base: Reg,
998 table_data: &TableData,
999 ) -> Result<M::Address> {
1000 let bound = self.context.any_gpr(self.masm)?;
1001 let tmp = self.context.any_gpr(self.masm)?;
1002 let ptr_size: OperandSize = self.env.ptr_type().try_into()?;
1003
1004 if let Some(offset) = table_data.import_from {
1005 // If the table data declares a particular offset base,
1006 // load the address into a register to further use it as
1007 // the table address.
1008 self.masm
1009 .load_ptr(self.masm.address_at_vmctx(offset)?, writable!(base))?;
1010 } else {
1011 // Else, simply move the vmctx register into the addr register as
1012 // the base to calculate the table address.
1013 self.masm.mov(writable!(base), vmctx!(M).into(), ptr_size)?;
1014 };
1015
1016 // OOB check.
1017 let bound_addr = self
1018 .masm
1019 .address_at_reg(base, table_data.current_elems_offset)?;
1020 let bound_size = table_data.current_elements_size;
1021 self.masm.load(bound_addr, writable!(bound), bound_size)?;
1022 self.masm.cmp(index, bound.into(), bound_size)?;
1023 self.masm
1024 .trapif(IntCmpKind::GeU, TRAP_TABLE_OUT_OF_BOUNDS)?;
1025
1026 // Move the index into the scratch register to calculate the table
1027 // element address.
1028 // Moving the value of the index register to the scratch register
1029 // also avoids overwriting the context of the index register.
1030 self.masm.with_scratch::<IntScratch, _>(|masm, scratch| {
1031 masm.mov(scratch.writable(), index.into(), bound_size)?;
1032 masm.mul(
1033 scratch.writable(),
1034 scratch.inner(),
1035 RegImm::i32(table_data.element_size.bytes() as i32),
1036 table_data.element_size,
1037 )?;
1038 masm.load_ptr(
1039 masm.address_at_reg(base, table_data.offset)?,
1040 writable!(base),
1041 )?;
1042 // Copy the value of the table base into a temporary register
1043 // so that we can use it later in case of a misspeculation.
1044 masm.mov(writable!(tmp), base.into(), ptr_size)?;
1045 // Calculate the address of the table element.
1046 masm.add(writable!(base), base, scratch.inner().into(), ptr_size)
1047 })?;
1048 if self.env.table_access_spectre_mitigation() {
1049 // Perform a bounds check and override the value of the
1050 // table element address in case the index is out of bounds.
1051 self.masm.cmp(index, bound.into(), bound_size)?;
1052 self.masm
1053 .cmov(writable!(base), tmp, IntCmpKind::GeU, ptr_size)?;
1054 }
1055 self.context.free_reg(bound);
1056 self.context.free_reg(tmp);
1057 self.masm.address_at_reg(base, 0)
1058 }
1059
1060 /// Retrieves the size of the table, pushing the result to the value stack.
emit_compute_table_size(&mut self, table_data: &TableData) -> Result<()>1061 pub fn emit_compute_table_size(&mut self, table_data: &TableData) -> Result<()> {
1062 let size = self.context.any_gpr(self.masm)?;
1063 let ptr_size: OperandSize = self.env.ptr_type().try_into()?;
1064
1065 self.masm.with_scratch::<IntScratch, _>(|masm, scratch| {
1066 if let Some(offset) = table_data.import_from {
1067 masm.load_ptr(masm.address_at_vmctx(offset)?, scratch.writable())?;
1068 } else {
1069 masm.mov(scratch.writable(), vmctx!(M).into(), ptr_size)?;
1070 };
1071
1072 let size_addr =
1073 masm.address_at_reg(scratch.inner(), table_data.current_elems_offset)?;
1074 masm.load(size_addr, writable!(size), table_data.current_elements_size)
1075 })?;
1076
1077 let dst = TypedReg::new(table_data.index_type(), size);
1078 self.context.stack.push(dst.into());
1079 Ok(())
1080 }
1081
1082 /// Retrieves the size of the memory, pushing the result to the value stack.
emit_compute_memory_size(&mut self, heap_data: &HeapData) -> Result<()>1083 pub fn emit_compute_memory_size(&mut self, heap_data: &HeapData) -> Result<()> {
1084 let size_reg = self.context.any_gpr(self.masm)?;
1085
1086 self.masm.with_scratch::<IntScratch, _>(|masm, scratch| {
1087 let base = if let Some(offset) = heap_data.import_from {
1088 masm.load_ptr(masm.address_at_vmctx(offset)?, scratch.writable())?;
1089 scratch.inner()
1090 } else {
1091 vmctx!(M)
1092 };
1093
1094 let size_addr = masm.address_at_reg(base, heap_data.current_length_offset)?;
1095 masm.load_ptr(size_addr, writable!(size_reg))
1096 })?;
1097 // Emit a shift to get the size in pages rather than in bytes.
1098 let dst = TypedReg::new(heap_data.index_type(), size_reg);
1099 let pow = heap_data.memory.page_size_log2;
1100 self.masm.shift_ir(
1101 writable!(dst.reg),
1102 Imm::i32(pow as i32),
1103 dst.into(),
1104 ShiftKind::ShrU,
1105 self.env.ptr_type().try_into()?,
1106 )?;
1107 self.context.stack.push(dst.into());
1108 Ok(())
1109 }
1110
1111 /// Checks if fuel consumption is enabled and emits a series of instructions
1112 /// that check the current fuel usage by performing a zero-comparison with
1113 /// the number of units stored in `VMStoreContext`.
maybe_emit_fuel_check(&mut self) -> Result<()>1114 pub fn maybe_emit_fuel_check(&mut self) -> Result<()> {
1115 if !self.tunables.consume_fuel {
1116 return Ok(());
1117 }
1118
1119 self.emit_fuel_increment()?;
1120 let out_of_fuel = self.env.builtins.out_of_gas::<M::ABI>()?;
1121 let fuel_reg = self.context.without::<Result<Reg>, M, _>(
1122 &out_of_fuel.sig().regs,
1123 self.masm,
1124 |cx, masm| cx.any_gpr(masm),
1125 )??;
1126
1127 self.emit_load_fuel_consumed(fuel_reg)?;
1128
1129 // The continuation label if the current fuel is under the limit.
1130 let continuation = self.masm.get_label()?;
1131
1132 // Spill locals and registers to avoid conflicts at the out-of-fuel
1133 // control flow merge.
1134 self.context.spill(self.masm)?;
1135 // Fuel is stored as a negative i64, so if the number is less than zero,
1136 // we're still under the fuel limits.
1137 self.masm.branch(
1138 IntCmpKind::LtS,
1139 fuel_reg,
1140 RegImm::i64(0),
1141 continuation,
1142 OperandSize::S64,
1143 )?;
1144 // Out-of-fuel branch.
1145 FnCall::emit::<M>(
1146 &mut self.env,
1147 self.masm,
1148 &mut self.context,
1149 Callee::Builtin(out_of_fuel.clone()),
1150 )?;
1151 self.context.pop_and_free(self.masm)?;
1152
1153 // Under fuel limits branch.
1154 self.masm.bind(continuation)?;
1155 self.context.free_reg(fuel_reg);
1156
1157 Ok(())
1158 }
1159
1160 /// Emits a series of instructions that load the `fuel_consumed` field from
1161 /// `VMStoreContext`.
emit_load_fuel_consumed(&mut self, fuel_reg: Reg) -> Result<()>1162 fn emit_load_fuel_consumed(&mut self, fuel_reg: Reg) -> Result<()> {
1163 let store_context_offset = self.env.vmoffsets.ptr.vmctx_store_context();
1164 let fuel_offset = self.env.vmoffsets.ptr.vmstore_context_fuel_consumed();
1165 self.masm.load_ptr(
1166 self.masm
1167 .address_at_vmctx(u32::from(store_context_offset))?,
1168 writable!(fuel_reg),
1169 )?;
1170
1171 self.masm.load(
1172 self.masm.address_at_reg(fuel_reg, u32::from(fuel_offset))?,
1173 writable!(fuel_reg),
1174 // Fuel is an i64.
1175 OperandSize::S64,
1176 )
1177 }
1178
1179 /// Checks if epoch interruption is configured and emits a series of
1180 /// instructions that check the current epoch against its deadline.
maybe_emit_epoch_check(&mut self) -> Result<()>1181 pub fn maybe_emit_epoch_check(&mut self) -> Result<()> {
1182 if !self.tunables.epoch_interruption {
1183 return Ok(());
1184 }
1185
1186 // The continuation branch if the current epoch hasn't reached the
1187 // configured deadline.
1188 let cont = self.masm.get_label()?;
1189 let new_epoch = self.env.builtins.new_epoch::<M::ABI>()?;
1190
1191 // Checks for runtime limits (e.g., fuel, epoch) are special since they
1192 // require inserting arbitrary function calls and control flow.
1193 // Special care must be taken to ensure that all invariants are met. In
1194 // this case, since `new_epoch` takes an argument and returns a value,
1195 // we must ensure that any registers used to hold the current epoch
1196 // value and deadline are not going to be needed later on by the
1197 // function call.
1198 let (epoch_deadline_reg, epoch_counter_reg) =
1199 self.context.without::<Result<(Reg, Reg)>, M, _>(
1200 &new_epoch.sig().regs,
1201 self.masm,
1202 |cx, masm| Ok((cx.any_gpr(masm)?, cx.any_gpr(masm)?)),
1203 )??;
1204
1205 self.emit_load_epoch_deadline_and_counter(epoch_deadline_reg, epoch_counter_reg)?;
1206
1207 // Spill locals and registers to avoid conflicts at the control flow
1208 // merge below.
1209 self.context.spill(self.masm)?;
1210 self.masm.branch(
1211 IntCmpKind::LtU,
1212 epoch_counter_reg,
1213 RegImm::reg(epoch_deadline_reg),
1214 cont,
1215 OperandSize::S64,
1216 )?;
1217 // Epoch deadline reached branch.
1218 FnCall::emit::<M>(
1219 &mut self.env,
1220 self.masm,
1221 &mut self.context,
1222 Callee::Builtin(new_epoch.clone()),
1223 )?;
1224 // `new_epoch` returns the new deadline. However we don't
1225 // perform any caching, so we simply drop this value.
1226 self.visit_drop()?;
1227
1228 // Under epoch deadline branch.
1229 self.masm.bind(cont)?;
1230
1231 self.context.free_reg(epoch_deadline_reg);
1232 self.context.free_reg(epoch_counter_reg);
1233 Ok(())
1234 }
1235
emit_load_epoch_deadline_and_counter( &mut self, epoch_deadline_reg: Reg, epoch_counter_reg: Reg, ) -> Result<()>1236 fn emit_load_epoch_deadline_and_counter(
1237 &mut self,
1238 epoch_deadline_reg: Reg,
1239 epoch_counter_reg: Reg,
1240 ) -> Result<()> {
1241 let epoch_ptr_offset = self.env.vmoffsets.ptr.vmctx_epoch_ptr();
1242 let store_context_offset = self.env.vmoffsets.ptr.vmctx_store_context();
1243 let epoch_deadline_offset = self.env.vmoffsets.ptr.vmstore_context_epoch_deadline();
1244
1245 // Load the current epoch value into `epoch_counter_var`.
1246 self.masm.load_ptr(
1247 self.masm.address_at_vmctx(u32::from(epoch_ptr_offset))?,
1248 writable!(epoch_counter_reg),
1249 )?;
1250
1251 // `epoch_deadline_var` contains the address of the value, so we need
1252 // to extract it.
1253 self.masm.load(
1254 self.masm.address_at_reg(epoch_counter_reg, 0)?,
1255 writable!(epoch_counter_reg),
1256 OperandSize::S64,
1257 )?;
1258
1259 // Load the `VMStoreContext`.
1260 self.masm.load_ptr(
1261 self.masm
1262 .address_at_vmctx(u32::from(store_context_offset))?,
1263 writable!(epoch_deadline_reg),
1264 )?;
1265
1266 self.masm.load(
1267 self.masm
1268 .address_at_reg(epoch_deadline_reg, u32::from(epoch_deadline_offset))?,
1269 writable!(epoch_deadline_reg),
1270 // The deadline value is a u64.
1271 OperandSize::S64,
1272 )
1273 }
1274
1275 /// Increments the fuel consumed in `VMStoreContext` by flushing
1276 /// `self.fuel_consumed` to memory.
emit_fuel_increment(&mut self) -> Result<()>1277 fn emit_fuel_increment(&mut self) -> Result<()> {
1278 let fuel_at_point = std::mem::replace(&mut self.fuel_consumed, 0);
1279 if fuel_at_point == 0 {
1280 return Ok(());
1281 }
1282
1283 let store_context_offset = self.env.vmoffsets.ptr.vmctx_store_context();
1284 let fuel_offset = self.env.vmoffsets.ptr.vmstore_context_fuel_consumed();
1285 let limits_reg = self.context.any_gpr(self.masm)?;
1286
1287 // Load `VMStoreContext` into the `limits_reg` reg.
1288 self.masm.load_ptr(
1289 self.masm
1290 .address_at_vmctx(u32::from(store_context_offset))?,
1291 writable!(limits_reg),
1292 )?;
1293
1294 self.masm.with_scratch::<IntScratch, _>(|masm, scratch| {
1295 // Load the fuel consumed at point into the scratch register.
1296 masm.load(
1297 masm.address_at_reg(limits_reg, u32::from(fuel_offset))?,
1298 scratch.writable(),
1299 OperandSize::S64,
1300 )?;
1301
1302 // Add the fuel consumed at point with the value in the scratch
1303 // register.
1304 masm.add(
1305 scratch.writable(),
1306 scratch.inner(),
1307 RegImm::i64(fuel_at_point),
1308 OperandSize::S64,
1309 )?;
1310
1311 // Store the updated fuel consumed to `VMStoreContext`.
1312 masm.store(
1313 scratch.inner().into(),
1314 masm.address_at_reg(limits_reg, u32::from(fuel_offset))?,
1315 OperandSize::S64,
1316 )
1317 })?;
1318
1319 self.context.free_reg(limits_reg);
1320
1321 Ok(())
1322 }
1323
1324 /// Hook to handle fuel before visiting an operator.
fuel_before_visit_op(&mut self, op: &Operator) -> Result<()>1325 fn fuel_before_visit_op(&mut self, op: &Operator) -> Result<()> {
1326 if !self.context.reachable {
1327 // `self.fuel_consumed` must be correctly flushed to memory when
1328 // entering an unreachable state.
1329 ensure!(self.fuel_consumed == 0, CodeGenError::illegal_fuel_state())
1330 }
1331
1332 // Generally, most instructions require 1 fuel unit.
1333 //
1334 // However, there are exceptions, which are detailed in the code below.
1335 // Note that the fuel accounting semantics align with those of
1336 // Cranelift; for further information, refer to
1337 // `crates/cranelift/src/func_environ.rs`.
1338 //
1339 // The primary distinction between the two implementations is that Winch
1340 // does not utilize a local-based cache to track fuel consumption.
1341 // Instead, each increase in fuel necessitates loading from and storing
1342 // to memory.
1343 //
1344 // Memory traffic will undoubtedly impact runtime performance. One
1345 // potential optimization is to designate a register as non-allocatable,
1346 // when fuel consumption is enabled, effectively using it as a local
1347 // fuel cache.
1348 self.fuel_consumed += self.tunables.operator_cost.cost(op);
1349
1350 match op {
1351 Operator::Unreachable
1352 | Operator::Loop { .. }
1353 | Operator::If { .. }
1354 | Operator::Else { .. }
1355 | Operator::Br { .. }
1356 | Operator::BrIf { .. }
1357 | Operator::BrTable { .. }
1358 | Operator::End
1359 | Operator::Return
1360 | Operator::CallIndirect { .. }
1361 | Operator::Call { .. }
1362 | Operator::ReturnCall { .. }
1363 | Operator::ReturnCallIndirect { .. } => self.emit_fuel_increment(),
1364 _ => Ok(()),
1365 }
1366 }
1367
1368 // Hook to handle source location mapping before visiting an operator.
source_location_before_visit_op(&mut self, offset: usize) -> Result<()>1369 fn source_location_before_visit_op(&mut self, offset: usize) -> Result<()> {
1370 let loc = SourceLoc::new(offset as u32);
1371 let rel = self.source_loc_from(loc);
1372 self.source_location.current = self.masm.start_source_loc(rel)?;
1373 Ok(())
1374 }
1375
1376 // Hook to handle source location mapping after visiting an operator.
source_location_after_visit_op(&mut self) -> Result<()>1377 fn source_location_after_visit_op(&mut self) -> Result<()> {
1378 // Because in Winch binary emission is done in a single pass
1379 // and because the MachBuffer performs optimizations during
1380 // emission, we have to be careful when calling
1381 // [`MacroAssembler::end_source_location`] to avoid breaking the
1382 // invariant that checks that the end [CodeOffset] must be equal
1383 // or greater than the start [CodeOffset].
1384 if self.masm.current_code_offset()? >= self.source_location.current.0 {
1385 self.masm.end_source_loc()?;
1386 }
1387
1388 Ok(())
1389 }
1390
emit_atomic_rmw( &mut self, arg: &MemArg, op: RmwOp, size: OperandSize, extend: Option<Extend<Zero>>, ) -> Result<()>1391 pub(crate) fn emit_atomic_rmw(
1392 &mut self,
1393 arg: &MemArg,
1394 op: RmwOp,
1395 size: OperandSize,
1396 extend: Option<Extend<Zero>>,
1397 ) -> Result<()> {
1398 // We need to pop-push the operand to compute the address before passing control over to
1399 // masm, because some architectures may have specific requirements for the registers used
1400 // in some atomic operations.
1401 let operand = self.context.pop_to_reg(self.masm, None)?;
1402 if let Some(addr) = self.emit_compute_heap_address_align_checked(arg, size)? {
1403 let src = self.masm.address_at_reg(addr, 0)?;
1404 self.context.stack.push(operand.into());
1405 self.masm
1406 .atomic_rmw(&mut self.context, src, size, op, UNTRUSTED_FLAGS, extend)?;
1407 self.context.free_reg(addr);
1408 }
1409
1410 Ok(())
1411 }
1412
emit_atomic_cmpxchg( &mut self, arg: &MemArg, size: OperandSize, extend: Option<Extend<Zero>>, ) -> Result<()>1413 pub(crate) fn emit_atomic_cmpxchg(
1414 &mut self,
1415 arg: &MemArg,
1416 size: OperandSize,
1417 extend: Option<Extend<Zero>>,
1418 ) -> Result<()> {
1419 // Emission for this instruction is a bit trickier. The address for the CAS is the 3rd from
1420 // the top of the stack, and we must emit instruction to compute the actual address with
1421 // `emit_compute_heap_address_align_checked`, while we still have access to self. However,
1422 // some ISAs have requirements with regard to the registers used for some arguments, so we
1423 // need to pass the context to the masm. To solve this issue, we pop the two first
1424 // arguments from the stack, compute the address, push back the arguments, and hand over
1425 // the control to masm. The implementer of `atomic_cas` can expect to find `expected` and
1426 // `replacement` at the top the context's stack.
1427
1428 // pop the args
1429 let replacement = self.context.pop_to_reg(self.masm, None)?;
1430 let expected = self.context.pop_to_reg(self.masm, None)?;
1431
1432 if let Some(addr) = self.emit_compute_heap_address_align_checked(arg, size)? {
1433 // push back the args
1434 self.context.stack.push(expected.into());
1435 self.context.stack.push(replacement.into());
1436
1437 let src = self.masm.address_at_reg(addr, 0)?;
1438 self.masm
1439 .atomic_cas(&mut self.context, src, size, UNTRUSTED_FLAGS, extend)?;
1440
1441 self.context.free_reg(addr);
1442 }
1443 Ok(())
1444 }
1445
1446 #[cfg(not(feature = "threads"))]
emit_atomic_wait(&mut self, _arg: &MemArg, _kind: AtomicWaitKind) -> Result<()>1447 pub fn emit_atomic_wait(&mut self, _arg: &MemArg, _kind: AtomicWaitKind) -> Result<()> {
1448 Err(CodeGenError::unimplemented_wasm_instruction().into())
1449 }
1450
1451 /// Emit the sequence of instruction for a `memory.atomic.wait*`.
1452 #[cfg(feature = "threads")]
emit_atomic_wait(&mut self, arg: &MemArg, kind: AtomicWaitKind) -> Result<()>1453 pub fn emit_atomic_wait(&mut self, arg: &MemArg, kind: AtomicWaitKind) -> Result<()> {
1454 // The `memory_atomic_wait*` builtins expect the following arguments:
1455 // - `memory`, as u32
1456 // - `address`, as u64
1457 // - `expected`, as either u64 or u32
1458 // - `timeout`, as u64
1459 // At this point our stack only contains the `timeout`, the `expected` and the address, so
1460 // we need to:
1461 // - insert the memory as the first argument
1462 // - compute the actual memory offset from the `MemArg`, if necessary.
1463 // Note that the builtin function performs the alignment and bounds checks for us, so we
1464 // don't need to emit that.
1465
1466 let timeout = self.context.pop_to_reg(self.masm, None)?;
1467 let expected = self.context.pop_to_reg(self.masm, None)?;
1468 let addr = self.context.pop_to_reg(self.masm, None)?;
1469
1470 // Put the target memory index as the first argument.
1471 let stack_len = self.context.stack.len();
1472 let builtin = match kind {
1473 AtomicWaitKind::Wait32 => self.env.builtins.memory_atomic_wait32::<M::ABI>()?,
1474 AtomicWaitKind::Wait64 => self.env.builtins.memory_atomic_wait64::<M::ABI>()?,
1475 };
1476 let builtin = self.prepare_builtin_defined_memory_arg(
1477 MemoryIndex::from_u32(arg.memory),
1478 stack_len,
1479 builtin,
1480 )?;
1481
1482 if arg.offset != 0 {
1483 self.masm.checked_uadd(
1484 writable!(addr.reg),
1485 addr.reg,
1486 Imm::i64(arg.offset as i64),
1487 OperandSize::S64,
1488 TrapCode::HEAP_OUT_OF_BOUNDS,
1489 )?;
1490 }
1491
1492 self.context
1493 .stack
1494 .push(TypedReg::new(WasmValType::I64, addr.reg).into());
1495 self.context.stack.push(expected.into());
1496 self.context.stack.push(timeout.into());
1497
1498 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, builtin)?;
1499
1500 Ok(())
1501 }
1502
1503 #[cfg(not(feature = "threads"))]
emit_atomic_notify(&mut self, _arg: &MemArg) -> Result<()>1504 pub fn emit_atomic_notify(&mut self, _arg: &MemArg) -> Result<()> {
1505 Err(CodeGenError::unimplemented_wasm_instruction().into())
1506 }
1507
1508 #[cfg(feature = "threads")]
emit_atomic_notify(&mut self, arg: &MemArg) -> Result<()>1509 pub fn emit_atomic_notify(&mut self, arg: &MemArg) -> Result<()> {
1510 // The memory `memory_atomic_notify` builtin expects the following arguments:
1511 // - `memory`, as u32
1512 // - `address`, as u64
1513 // - `count`: as u32
1514 // At this point our stack only contains the `count` and the `address`, so we need to:
1515 // - insert the memory as the first argument
1516 // - compute the actual memory offset from the `MemArg`, if necessary.
1517 // Note that the builtin function performs the alignment and bounds checks for us, so we
1518 // don't need to emit that.
1519
1520 // pop the arguments from the stack.
1521 let count = self.context.pop_to_reg(self.masm, None)?;
1522 let addr = self.context.pop_to_reg(self.masm, None)?;
1523
1524 // Put the target memory index as the first argument.
1525 let builtin = self.env.builtins.memory_atomic_notify::<M::ABI>()?;
1526 let stack_len = self.context.stack.len();
1527 let builtin = self.prepare_builtin_defined_memory_arg(
1528 MemoryIndex::from_u32(arg.memory),
1529 stack_len,
1530 builtin,
1531 )?;
1532
1533 if arg.offset != 0 {
1534 self.masm.checked_uadd(
1535 writable!(addr.reg),
1536 addr.reg,
1537 Imm::i64(arg.offset as i64),
1538 OperandSize::S64,
1539 TrapCode::HEAP_OUT_OF_BOUNDS,
1540 )?;
1541 }
1542
1543 // push remaining arguments.
1544 self.context
1545 .stack
1546 .push(TypedReg::new(WasmValType::I64, addr.reg).into());
1547 self.context.stack.push(count.into());
1548
1549 FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, builtin)?;
1550
1551 Ok(())
1552 }
1553
prepare_builtin_defined_memory_arg( &mut self, mem: MemoryIndex, defined_index_at: usize, builtin: BuiltinFunction, ) -> Result<Callee>1554 pub fn prepare_builtin_defined_memory_arg(
1555 &mut self,
1556 mem: MemoryIndex,
1557 defined_index_at: usize,
1558 builtin: BuiltinFunction,
1559 ) -> Result<Callee> {
1560 match self.env.translation.module.defined_memory_index(mem) {
1561 // This memory is defined in this module, so the vmctx is this
1562 // module's vmctx and the memory index is `defined` as returned here.
1563 Some(defined) => {
1564 self.context
1565 .stack
1566 .insert_many(defined_index_at, &[defined.as_u32().try_into()?]);
1567 Ok(Callee::Builtin(builtin))
1568 }
1569
1570 // This memory is not defined in this module, so the defined index
1571 // is loaded from the `VMMemoryImport` and the vmctx is loaded from
1572 // the vmctx itself.
1573 None => {
1574 let vmimport = self.env.vmoffsets.vmctx_vmmemory_import(mem);
1575 let vmctx_offset = vmimport + u32::from(self.env.vmoffsets.vmmemory_import_vmctx());
1576 let index_offset = vmimport + u32::from(self.env.vmoffsets.vmmemory_import_index());
1577 let index_addr = self.masm.address_at_vmctx(index_offset)?;
1578 let index_dst = self.context.reg_for_class(RegClass::Int, self.masm)?;
1579 self.masm
1580 .load(index_addr, writable!(index_dst), OperandSize::S32)?;
1581 self.context
1582 .stack
1583 .insert_many(defined_index_at, &[Val::reg(index_dst, WasmValType::I32)]);
1584 Ok(Callee::BuiltinWithDifferentVmctx(builtin, vmctx_offset))
1585 }
1586 }
1587 }
1588
1589 /// Same as `prepare_builtin_defined_memory_arg`, but for tables.
prepare_builtin_defined_table_arg( &mut self, table: TableIndex, defined_index_at: usize, builtin: BuiltinFunction, ) -> Result<Callee>1590 pub fn prepare_builtin_defined_table_arg(
1591 &mut self,
1592 table: TableIndex,
1593 defined_index_at: usize,
1594 builtin: BuiltinFunction,
1595 ) -> Result<Callee> {
1596 match self.env.translation.module.defined_table_index(table) {
1597 Some(defined) => {
1598 self.context
1599 .stack
1600 .insert_many(defined_index_at, &[defined.as_u32().try_into()?]);
1601 Ok(Callee::Builtin(builtin))
1602 }
1603 None => {
1604 let vmimport = self.env.vmoffsets.vmctx_vmtable_import(table);
1605 let vmctx_offset = vmimport + u32::from(self.env.vmoffsets.vmtable_import_vmctx());
1606 let index_offset = vmimport + u32::from(self.env.vmoffsets.vmtable_import_index());
1607 let index_addr = self.masm.address_at_vmctx(index_offset)?;
1608 let index_dst = self.context.reg_for_class(RegClass::Int, self.masm)?;
1609 self.masm
1610 .load(index_addr, writable!(index_dst), OperandSize::S32)?;
1611 self.context
1612 .stack
1613 .insert_many(defined_index_at, &[Val::reg(index_dst, WasmValType::I32)]);
1614 Ok(Callee::BuiltinWithDifferentVmctx(builtin, vmctx_offset))
1615 }
1616 }
1617 }
1618 }
1619
1620 /// Returns the index of the [`ControlStackFrame`] for the given
1621 /// depth.
control_index(depth: u32, control_length: usize) -> Result<usize>1622 pub fn control_index(depth: u32, control_length: usize) -> Result<usize> {
1623 (control_length - 1)
1624 .checked_sub(depth as usize)
1625 .ok_or_else(|| format_err!(CodeGenError::control_frame_expected()))
1626 }
1627