163d482c8SFrank Emrich //! This module contains the runtime components of the implementation of the
263d482c8SFrank Emrich //! stack switching proposal.
363d482c8SFrank Emrich
463d482c8SFrank Emrich mod stack;
563d482c8SFrank Emrich
663d482c8SFrank Emrich use core::{marker::PhantomPinned, ptr::NonNull};
763d482c8SFrank Emrich
863d482c8SFrank Emrich pub use stack::*;
963d482c8SFrank Emrich
1063d482c8SFrank Emrich /// A continuation object is a handle to a continuation reference
1163d482c8SFrank Emrich /// (i.e. an actual stack). A continuation object only be consumed
1263d482c8SFrank Emrich /// once. The linearity is checked dynamically in the generated code
1363d482c8SFrank Emrich /// by comparing the revision witness embedded in the pointer to the
1463d482c8SFrank Emrich /// actual revision counter on the continuation reference.
1563d482c8SFrank Emrich ///
1663d482c8SFrank Emrich /// In the optimized implementation, the continuation logically
1763d482c8SFrank Emrich /// represented by a VMContObj not only encompasses the pointed-to
1863d482c8SFrank Emrich /// VMContRef, but also all of its parents:
1963d482c8SFrank Emrich ///
2063d482c8SFrank Emrich /// ```text
2163d482c8SFrank Emrich ///
2263d482c8SFrank Emrich /// +----------------+
2363d482c8SFrank Emrich /// +-->| VMContRef |
2463d482c8SFrank Emrich /// | +----------------+
2563d482c8SFrank Emrich /// | ^
2663d482c8SFrank Emrich /// | | parent
2763d482c8SFrank Emrich /// | |
2863d482c8SFrank Emrich /// | +----------------+
2963d482c8SFrank Emrich /// | | VMContRef |
3063d482c8SFrank Emrich /// | +----------------+
3163d482c8SFrank Emrich /// | ^
3263d482c8SFrank Emrich /// | | parent
3363d482c8SFrank Emrich /// last ancestor | |
3463d482c8SFrank Emrich /// | +----------------+
3563d482c8SFrank Emrich /// +---| VMContRef | <-- VMContObj
3663d482c8SFrank Emrich /// +----------------+
3763d482c8SFrank Emrich /// ```
3863d482c8SFrank Emrich ///
3963d482c8SFrank Emrich /// For performance reasons, the VMContRef at the bottom of this chain
4063d482c8SFrank Emrich /// (i.e., the one pointed to by the VMContObj) has a pointer to the
4163d482c8SFrank Emrich /// other end of the chain (i.e., its last ancestor).
42a631d20aSPaul Osborne #[repr(C)]
4363d482c8SFrank Emrich #[derive(Debug, Clone, Copy)]
4463d482c8SFrank Emrich pub struct VMContObj {
4563d482c8SFrank Emrich pub contref: NonNull<VMContRef>,
46a631d20aSPaul Osborne pub revision: usize,
4763d482c8SFrank Emrich }
4863d482c8SFrank Emrich
4963d482c8SFrank Emrich impl VMContObj {
new(contref: NonNull<VMContRef>, revision: usize) -> Self50a631d20aSPaul Osborne pub fn new(contref: NonNull<VMContRef>, revision: usize) -> Self {
5163d482c8SFrank Emrich Self { contref, revision }
5263d482c8SFrank Emrich }
5363d482c8SFrank Emrich
5463d482c8SFrank Emrich /// Construction a VMContinuationObject from a pointer and revision
5563d482c8SFrank Emrich ///
5663d482c8SFrank Emrich /// The `contref` pointer may be null in which case None will be returned.
5763d482c8SFrank Emrich ///
5863d482c8SFrank Emrich /// # Safety
5963d482c8SFrank Emrich ///
6063d482c8SFrank Emrich /// Behavior will be undefined if a pointer to data that is not a
6163d482c8SFrank Emrich /// VMContRef is provided.
from_raw_parts(contref: *mut u8, revision: usize) -> Option<Self>62a631d20aSPaul Osborne pub unsafe fn from_raw_parts(contref: *mut u8, revision: usize) -> Option<Self> {
6363d482c8SFrank Emrich NonNull::new(contref.cast::<VMContRef>()).map(|contref| Self::new(contref, revision))
6463d482c8SFrank Emrich }
6563d482c8SFrank Emrich }
6663d482c8SFrank Emrich
6763d482c8SFrank Emrich unsafe impl Send for VMContObj {}
6863d482c8SFrank Emrich unsafe impl Sync for VMContObj {}
6963d482c8SFrank Emrich
7063d482c8SFrank Emrich /// This type is used to save (and subsequently restore) a subset of the data in
7163d482c8SFrank Emrich /// `VMStoreContext`. See documentation of `VMStackChain` for the exact uses.
7263d482c8SFrank Emrich #[repr(C)]
7363d482c8SFrank Emrich #[derive(Debug, Default, Clone)]
7463d482c8SFrank Emrich pub struct VMStackLimits {
7563d482c8SFrank Emrich /// Saved version of `stack_limit` field of `VMStoreContext`
7663d482c8SFrank Emrich pub stack_limit: usize,
7763d482c8SFrank Emrich /// Saved version of `last_wasm_entry_fp` field of `VMStoreContext`
7863d482c8SFrank Emrich pub last_wasm_entry_fp: usize,
7963d482c8SFrank Emrich }
8063d482c8SFrank Emrich
8163d482c8SFrank Emrich /// This type represents "common" information that we need to save both for the
8263d482c8SFrank Emrich /// initial stack and each continuation.
8363d482c8SFrank Emrich #[repr(C)]
8463d482c8SFrank Emrich #[derive(Debug, Clone)]
8563d482c8SFrank Emrich pub struct VMCommonStackInformation {
8663d482c8SFrank Emrich /// Saves subset of `VMStoreContext` for this stack. See documentation of
8763d482c8SFrank Emrich /// `VMStackChain` for the exact uses.
8863d482c8SFrank Emrich pub limits: VMStackLimits,
8963d482c8SFrank Emrich /// For the initial stack, this field must only have one of the following values:
9063d482c8SFrank Emrich /// - Running
9163d482c8SFrank Emrich /// - Parent
9263d482c8SFrank Emrich pub state: VMStackState,
9363d482c8SFrank Emrich
9463d482c8SFrank Emrich /// Only in use when state is `Parent`. Otherwise, the list must be empty.
9563d482c8SFrank Emrich ///
9663d482c8SFrank Emrich /// Represents the handlers that this stack installed when resume-ing a
9763d482c8SFrank Emrich /// continuation.
9863d482c8SFrank Emrich ///
9963d482c8SFrank Emrich /// Note that for any resume instruction, we can re-order the handler
10063d482c8SFrank Emrich /// clauses without changing behavior such that all the suspend handlers
10163d482c8SFrank Emrich /// come first, followed by all the switch handler (while maintaining the
10263d482c8SFrank Emrich /// original ordering within the two groups).
10363d482c8SFrank Emrich /// Thus, we assume that the given resume instruction has the following
10463d482c8SFrank Emrich /// shape:
10563d482c8SFrank Emrich ///
10663d482c8SFrank Emrich /// (resume $ct
10763d482c8SFrank Emrich /// (on $tag_0 $block_0) ... (on $tag_{n-1} $block_{n-1})
10863d482c8SFrank Emrich /// (on $tag_n switch) ... (on $tag_m switch)
10963d482c8SFrank Emrich /// )
11063d482c8SFrank Emrich ///
11163d482c8SFrank Emrich /// On resume, the handler list is then filled with m + 1 (i.e., one per
11263d482c8SFrank Emrich /// handler clause) entries such that the i-th entry, using 0-based
11363d482c8SFrank Emrich /// indexing, is the identifier of $tag_i (represented as *mut
11463d482c8SFrank Emrich /// VMTagDefinition).
11563d482c8SFrank Emrich /// Further, `first_switch_handler_index` (see below) is set to n (i.e., the
11663d482c8SFrank Emrich /// 0-based index of the first switch handler).
11763d482c8SFrank Emrich ///
11863d482c8SFrank Emrich /// Note that the actual data buffer (i.e., the one `handler.data` points
11963d482c8SFrank Emrich /// to) is always allocated on the stack that this `CommonStackInformation`
12063d482c8SFrank Emrich /// struct describes.
12163d482c8SFrank Emrich pub handlers: VMHandlerList,
12263d482c8SFrank Emrich
12363d482c8SFrank Emrich /// Only used when state is `Parent`. See documentation of `handlers` above.
12463d482c8SFrank Emrich pub first_switch_handler_index: u32,
12563d482c8SFrank Emrich }
12663d482c8SFrank Emrich
12763d482c8SFrank Emrich impl VMCommonStackInformation {
12863d482c8SFrank Emrich /// Default value with state set to `Running`
running_default() -> Self12963d482c8SFrank Emrich pub fn running_default() -> Self {
13063d482c8SFrank Emrich Self {
13163d482c8SFrank Emrich limits: VMStackLimits::default(),
13263d482c8SFrank Emrich state: VMStackState::Running,
13363d482c8SFrank Emrich handlers: VMHandlerList::empty(),
13463d482c8SFrank Emrich first_switch_handler_index: 0,
13563d482c8SFrank Emrich }
13663d482c8SFrank Emrich }
13763d482c8SFrank Emrich }
13863d482c8SFrank Emrich
13963d482c8SFrank Emrich impl VMStackLimits {
14063d482c8SFrank Emrich /// Default value, but uses the given value for `stack_limit`.
with_stack_limit(stack_limit: usize) -> Self14163d482c8SFrank Emrich pub fn with_stack_limit(stack_limit: usize) -> Self {
14263d482c8SFrank Emrich Self {
14363d482c8SFrank Emrich stack_limit,
14463d482c8SFrank Emrich ..Default::default()
14563d482c8SFrank Emrich }
14663d482c8SFrank Emrich }
14763d482c8SFrank Emrich }
14863d482c8SFrank Emrich
14963d482c8SFrank Emrich #[repr(C)]
15063d482c8SFrank Emrich #[derive(Debug, Clone)]
15163d482c8SFrank Emrich /// Reference to a stack-allocated buffer ("array"), storing data of some type
15263d482c8SFrank Emrich /// `T`.
15363d482c8SFrank Emrich pub struct VMHostArray<T> {
15463d482c8SFrank Emrich /// Number of currently occupied slots.
15563d482c8SFrank Emrich pub length: u32,
15663d482c8SFrank Emrich /// Number of slots in the data buffer. Note that this is *not* the size of
15763d482c8SFrank Emrich /// the buffer in bytes!
15863d482c8SFrank Emrich pub capacity: u32,
15963d482c8SFrank Emrich /// The actual data buffer
16063d482c8SFrank Emrich pub data: *mut T,
16163d482c8SFrank Emrich }
16263d482c8SFrank Emrich
16363d482c8SFrank Emrich impl<T> VMHostArray<T> {
16463d482c8SFrank Emrich /// Creates empty `Array`
empty() -> Self16563d482c8SFrank Emrich pub fn empty() -> Self {
16663d482c8SFrank Emrich Self {
16763d482c8SFrank Emrich length: 0,
16863d482c8SFrank Emrich capacity: 0,
16963d482c8SFrank Emrich data: core::ptr::null_mut(),
17063d482c8SFrank Emrich }
17163d482c8SFrank Emrich }
17263d482c8SFrank Emrich
17363d482c8SFrank Emrich /// Makes `Array` empty.
clear(&mut self)17463d482c8SFrank Emrich pub fn clear(&mut self) {
17563d482c8SFrank Emrich *self = Self::empty();
17663d482c8SFrank Emrich }
17763d482c8SFrank Emrich }
17863d482c8SFrank Emrich
17963d482c8SFrank Emrich /// Type used for passing payloads to and from continuations. The actual type
18063d482c8SFrank Emrich /// argument should be wasmtime::runtime::vm::vmcontext::ValRaw, but we don't
18163d482c8SFrank Emrich /// have access to that here.
18263d482c8SFrank Emrich pub type VMPayloads = VMHostArray<u128>;
18363d482c8SFrank Emrich
18463d482c8SFrank Emrich /// Type for a list of handlers, represented by the handled tag. Thus, the
185*ab78bd82SHo Kim /// stored data is actually `*mut VMTagDefinition`, but we don't have access to
18663d482c8SFrank Emrich /// that here.
18763d482c8SFrank Emrich pub type VMHandlerList = VMHostArray<*mut u8>;
18863d482c8SFrank Emrich
18963d482c8SFrank Emrich /// The main type representing a continuation.
19063d482c8SFrank Emrich #[repr(C)]
19163d482c8SFrank Emrich pub struct VMContRef {
19263d482c8SFrank Emrich /// The `CommonStackInformation` of this continuation's stack.
19363d482c8SFrank Emrich pub common_stack_information: VMCommonStackInformation,
19463d482c8SFrank Emrich
19563d482c8SFrank Emrich /// The parent of this continuation, which may be another continuation, the
19663d482c8SFrank Emrich /// initial stack, or absent (in case of a suspended continuation).
19763d482c8SFrank Emrich pub parent_chain: VMStackChain,
19863d482c8SFrank Emrich
19963d482c8SFrank Emrich /// Only used if `common_stack_information.state` is `Suspended` or `Fresh`. In
20063d482c8SFrank Emrich /// that case, this points to the end of the stack chain (i.e., the
20163d482c8SFrank Emrich /// continuation in the parent chain whose own `parent_chain` field is
20263d482c8SFrank Emrich /// `VMStackChain::Absent`).
203e16780deSfuder.eth /// Note that this may be a pointer to itself (if the state is `Fresh`, this is always the case).
20463d482c8SFrank Emrich pub last_ancestor: *mut VMContRef,
20563d482c8SFrank Emrich
20663d482c8SFrank Emrich /// Revision counter.
207a631d20aSPaul Osborne pub revision: usize,
20863d482c8SFrank Emrich
20963d482c8SFrank Emrich /// The underlying stack.
21063d482c8SFrank Emrich pub stack: VMContinuationStack,
21163d482c8SFrank Emrich
21263d482c8SFrank Emrich /// Used to store only
21363d482c8SFrank Emrich /// 1. The arguments to the function passed to cont.new
21463d482c8SFrank Emrich /// 2. The return values of that function
21563d482c8SFrank Emrich ///
21663d482c8SFrank Emrich /// Note that the actual data buffer (i.e., the one `args.data` points
21763d482c8SFrank Emrich /// to) is always allocated on this continuation's stack.
21863d482c8SFrank Emrich pub args: VMPayloads,
21963d482c8SFrank Emrich
22063d482c8SFrank Emrich /// Once a continuation has been suspended (using suspend or switch),
22163d482c8SFrank Emrich /// this buffer is used to pass payloads to and from the continuation.
22263d482c8SFrank Emrich /// More concretely, it is used to
22363d482c8SFrank Emrich /// - Pass payloads from a suspend instruction to the corresponding handler.
22463d482c8SFrank Emrich /// - Pass payloads to a continuation using cont.bind or resume
22563d482c8SFrank Emrich /// - Pass payloads to the continuation being switched to when using switch.
22663d482c8SFrank Emrich ///
22763d482c8SFrank Emrich /// Note that the actual data buffer (i.e., the one `values.data` points
22863d482c8SFrank Emrich /// to) is always allocated on this continuation's stack.
22963d482c8SFrank Emrich pub values: VMPayloads,
23063d482c8SFrank Emrich
23163d482c8SFrank Emrich /// Tell the compiler that this structure has potential self-references
23263d482c8SFrank Emrich /// through the `last_ancestor` pointer.
23363d482c8SFrank Emrich _marker: core::marker::PhantomPinned,
23463d482c8SFrank Emrich }
23563d482c8SFrank Emrich
23663d482c8SFrank Emrich impl VMContRef {
fiber_stack(&self) -> &VMContinuationStack23763d482c8SFrank Emrich pub fn fiber_stack(&self) -> &VMContinuationStack {
23863d482c8SFrank Emrich &self.stack
23963d482c8SFrank Emrich }
24063d482c8SFrank Emrich
detach_stack(&mut self) -> VMContinuationStack24163d482c8SFrank Emrich pub fn detach_stack(&mut self) -> VMContinuationStack {
24263d482c8SFrank Emrich core::mem::replace(&mut self.stack, VMContinuationStack::unallocated())
24363d482c8SFrank Emrich }
24463d482c8SFrank Emrich
24563d482c8SFrank Emrich /// This is effectively a `Default` implementation, without calling it
24663d482c8SFrank Emrich /// so. Used to create `VMContRef`s when initializing pooling allocator.
empty() -> Self24763d482c8SFrank Emrich pub fn empty() -> Self {
24863d482c8SFrank Emrich let limits = VMStackLimits::with_stack_limit(Default::default());
24963d482c8SFrank Emrich let state = VMStackState::Fresh;
25063d482c8SFrank Emrich let handlers = VMHandlerList::empty();
25163d482c8SFrank Emrich let common_stack_information = VMCommonStackInformation {
25263d482c8SFrank Emrich limits,
25363d482c8SFrank Emrich state,
25463d482c8SFrank Emrich handlers,
25563d482c8SFrank Emrich first_switch_handler_index: 0,
25663d482c8SFrank Emrich };
25763d482c8SFrank Emrich let parent_chain = VMStackChain::Absent;
25863d482c8SFrank Emrich let last_ancestor = core::ptr::null_mut();
25963d482c8SFrank Emrich let stack = VMContinuationStack::unallocated();
26063d482c8SFrank Emrich let args = VMPayloads::empty();
26163d482c8SFrank Emrich let values = VMPayloads::empty();
26263d482c8SFrank Emrich let revision = 0;
26363d482c8SFrank Emrich let _marker = PhantomPinned;
26463d482c8SFrank Emrich
26563d482c8SFrank Emrich Self {
26663d482c8SFrank Emrich common_stack_information,
26763d482c8SFrank Emrich parent_chain,
26863d482c8SFrank Emrich last_ancestor,
26963d482c8SFrank Emrich stack,
27063d482c8SFrank Emrich args,
27163d482c8SFrank Emrich values,
27263d482c8SFrank Emrich revision,
27363d482c8SFrank Emrich _marker,
27463d482c8SFrank Emrich }
27563d482c8SFrank Emrich }
27663d482c8SFrank Emrich }
27763d482c8SFrank Emrich
27863d482c8SFrank Emrich impl Drop for VMContRef {
drop(&mut self)27963d482c8SFrank Emrich fn drop(&mut self) {
28063d482c8SFrank Emrich // Note that continuation references do not own their parents, and we
28163d482c8SFrank Emrich // don't drop them here.
28263d482c8SFrank Emrich
28363d482c8SFrank Emrich // We would like to enforce the invariant that any continuation that
28463d482c8SFrank Emrich // was created for a cont.new (rather than, say, just living in a
28563d482c8SFrank Emrich // pool and never being touched), either ran to completion or was
28663d482c8SFrank Emrich // cancelled. But failing to do so should yield a custom error,
28763d482c8SFrank Emrich // instead of panicking here.
28863d482c8SFrank Emrich }
28963d482c8SFrank Emrich }
29063d482c8SFrank Emrich
29163d482c8SFrank Emrich // These are required so the WasmFX pooling allocator can store a Vec of
29263d482c8SFrank Emrich // `VMContRef`s.
29363d482c8SFrank Emrich unsafe impl Send for VMContRef {}
29463d482c8SFrank Emrich unsafe impl Sync for VMContRef {}
29563d482c8SFrank Emrich
29663d482c8SFrank Emrich /// Implements `cont.new` instructions (i.e., creation of continuations).
29763d482c8SFrank Emrich #[cfg(feature = "stack-switching")]
29863d482c8SFrank Emrich #[inline(always)]
cont_new( store: &mut dyn crate::vm::VMStore, instance: crate::store::InstanceId, func: *mut u8, param_count: u32, result_count: u32, ) -> crate::Result<*mut VMContRef>29963d482c8SFrank Emrich pub fn cont_new(
30063d482c8SFrank Emrich store: &mut dyn crate::vm::VMStore,
301900370bcSAlex Crichton instance: crate::store::InstanceId,
30263d482c8SFrank Emrich func: *mut u8,
30363d482c8SFrank Emrich param_count: u32,
30463d482c8SFrank Emrich result_count: u32,
30596e19700SNick Fitzgerald ) -> crate::Result<*mut VMContRef> {
306900370bcSAlex Crichton let instance = store.instance_mut(instance);
30763d482c8SFrank Emrich let caller_vmctx = instance.vmctx();
30863d482c8SFrank Emrich
30963d482c8SFrank Emrich let stack_size = store.engine().config().async_stack_size;
31063d482c8SFrank Emrich
31163d482c8SFrank Emrich let contref = store.allocate_continuation()?;
31263d482c8SFrank Emrich let contref = unsafe { contref.as_mut().unwrap() };
31363d482c8SFrank Emrich
31463d482c8SFrank Emrich let tsp = contref.stack.top().unwrap();
31563d482c8SFrank Emrich contref.parent_chain = VMStackChain::Absent;
31663d482c8SFrank Emrich // The continuation is fresh, which is a special case of being suspended.
31763d482c8SFrank Emrich // Thus we need to set the correct end of the continuation chain: itself.
31863d482c8SFrank Emrich contref.last_ancestor = contref;
31963d482c8SFrank Emrich
32063d482c8SFrank Emrich // The initialization function will allocate the actual args/return value buffer and
32163d482c8SFrank Emrich // update this object (if needed).
32263d482c8SFrank Emrich let contref_args_ptr = &mut contref.args as *mut _ as *mut VMHostArray<crate::ValRaw>;
32363d482c8SFrank Emrich
32463d482c8SFrank Emrich contref.stack.initialize(
32563d482c8SFrank Emrich func.cast::<crate::vm::VMFuncRef>(),
32663d482c8SFrank Emrich caller_vmctx.as_ptr(),
32763d482c8SFrank Emrich contref_args_ptr,
32863d482c8SFrank Emrich param_count,
32963d482c8SFrank Emrich result_count,
33063d482c8SFrank Emrich );
33163d482c8SFrank Emrich
33263d482c8SFrank Emrich // Now that the initial stack pointer was set by the initialization
33363d482c8SFrank Emrich // function, use it to determine stack limit.
33463d482c8SFrank Emrich let stack_pointer = contref.stack.control_context_stack_pointer();
335e16780deSfuder.eth // Same caveat regarding stack_limit here as described in
33663d482c8SFrank Emrich // `wasmtime::runtime::func::EntryStoreContext::enter_wasm`.
33763d482c8SFrank Emrich let wasm_stack_limit = core::cmp::max(
33863d482c8SFrank Emrich stack_pointer - store.engine().config().max_wasm_stack,
33963d482c8SFrank Emrich tsp as usize - stack_size,
34063d482c8SFrank Emrich );
34163d482c8SFrank Emrich let limits = VMStackLimits::with_stack_limit(wasm_stack_limit);
34263d482c8SFrank Emrich let csi = &mut contref.common_stack_information;
34363d482c8SFrank Emrich csi.state = VMStackState::Fresh;
34463d482c8SFrank Emrich csi.limits = limits;
34563d482c8SFrank Emrich
3462bac6574SAlex Crichton log::trace!("Created contref @ {contref:p}");
34763d482c8SFrank Emrich Ok(contref)
34863d482c8SFrank Emrich }
34963d482c8SFrank Emrich
35063d482c8SFrank Emrich /// This type represents a linked lists ("chain") of stacks, where the a
35163d482c8SFrank Emrich /// node's successor denotes its parent.
35263d482c8SFrank Emrich /// Additionally, a `CommonStackInformation` object is associated with
35363d482c8SFrank Emrich /// each stack in the list.
35463d482c8SFrank Emrich /// Here, a "stack" is one of the following:
35563d482c8SFrank Emrich /// - A continuation (i.e., created with cont.new).
35663d482c8SFrank Emrich /// - The initial stack. This is the stack that we were on when entering
35763d482c8SFrank Emrich /// Wasm (i.e., when executing
35863d482c8SFrank Emrich /// `crate::runtime::func::invoke_wasm_and_catch_traps`).
35963d482c8SFrank Emrich /// This stack never has a parent.
36063d482c8SFrank Emrich /// In terms of the memory allocation that this stack resides on, it will
36163d482c8SFrank Emrich /// usually be the main stack, but doesn't have to: If we are running
36263d482c8SFrank Emrich /// inside a continuation while executing a host call, which in turn
36363d482c8SFrank Emrich /// re-renters Wasm, the initial stack is actually the stack of that
36463d482c8SFrank Emrich /// continuation.
36563d482c8SFrank Emrich ///
36663d482c8SFrank Emrich /// Note that the linked list character of `VMStackChain` arises from the fact
36763d482c8SFrank Emrich /// that `VMStackChain::Continuation` variants have a pointer to a
36863d482c8SFrank Emrich /// `VMContRef`, which in turn has a `parent_chain` value of type
36963d482c8SFrank Emrich /// `VMStackChain`. This is how the stack chain reflects the parent-child
37063d482c8SFrank Emrich /// relationships between continuations/stacks. This also shows how the
37163d482c8SFrank Emrich /// initial stack (mentioned above) cannot have a parent.
37263d482c8SFrank Emrich ///
37363d482c8SFrank Emrich /// There are generally two uses of `VMStackChain`:
37463d482c8SFrank Emrich ///
37563d482c8SFrank Emrich /// 1. The `stack_chain` field in the `StoreOpaque` contains such a
37663d482c8SFrank Emrich /// chain of stacks, where the head of the list denotes the stack that is
37763d482c8SFrank Emrich /// currently executing (either a continuation or the initial stack). Note
37863d482c8SFrank Emrich /// that in this case, the linked list must contain 0 or more `Continuation`
37963d482c8SFrank Emrich /// elements, followed by a final `InitialStack` element. In particular,
38063d482c8SFrank Emrich /// this list always ends with `InitialStack` and never contains an `Absent`
38163d482c8SFrank Emrich /// variant.
38263d482c8SFrank Emrich ///
38363d482c8SFrank Emrich /// 2. When a continuation is suspended, its chain of parents eventually
38463d482c8SFrank Emrich /// ends with an `Absent` variant in its `parent_chain` field. Note that a
38563d482c8SFrank Emrich /// suspended continuation never appears in the stack chain in the
38663d482c8SFrank Emrich /// VMContext!
38763d482c8SFrank Emrich ///
38863d482c8SFrank Emrich ///
38963d482c8SFrank Emrich /// As mentioned before, each stack in a `VMStackChain` has a corresponding
39063d482c8SFrank Emrich /// `CommonStackInformation` object. For continuations, this is stored in
39163d482c8SFrank Emrich /// the `common_stack_information` field of the corresponding `VMContRef`.
39263d482c8SFrank Emrich /// For the initial stack, the `InitialStack` variant contains a pointer to
39363d482c8SFrank Emrich /// a `CommonStackInformation`. The latter will be allocated allocated on
39463d482c8SFrank Emrich /// the stack frame that executed by `invoke_wasm_and_catch_traps`.
39563d482c8SFrank Emrich ///
39663d482c8SFrank Emrich /// The following invariants hold for these `VMStackLimits` objects,
39763d482c8SFrank Emrich /// and the data in `VMStoreContext`.
39863d482c8SFrank Emrich ///
39963d482c8SFrank Emrich /// Currently executing stack: For the currently executing stack (i.e., the
40063d482c8SFrank Emrich /// stack that is at the head of the store's `stack_chain` list), the
40163d482c8SFrank Emrich /// associated `VMStackLimits` object contains stale/undefined data. Instead,
40263d482c8SFrank Emrich /// the live data describing the limits for the currently executing stack is
40363d482c8SFrank Emrich /// always maintained in `VMStoreContext`. Note that as a general rule
40463d482c8SFrank Emrich /// independently from any execution of continuations, the `last_wasm_exit*`
40563d482c8SFrank Emrich /// fields in the `VMStoreContext` contain undefined values while executing
40663d482c8SFrank Emrich /// wasm.
40763d482c8SFrank Emrich ///
40863d482c8SFrank Emrich /// Parents of currently executing stack: For stacks that appear in the tail
40963d482c8SFrank Emrich /// of the store's `stack_chain` list (i.e., stacks that are not currently
41063d482c8SFrank Emrich /// executing themselves, but are an ancestor of the currently executing
41163d482c8SFrank Emrich /// stack), we have the following: All the fields in the stack's
41263d482c8SFrank Emrich /// `VMStackLimits` are valid, describing the stack's stack limit, and
41363d482c8SFrank Emrich /// pointers where executing for that stack entered and exited WASM.
41463d482c8SFrank Emrich ///
41563d482c8SFrank Emrich /// Suspended continuations: For suspended continuations (including their
41663d482c8SFrank Emrich /// ancestors), we have the following. Note that the initial stack can never
41763d482c8SFrank Emrich /// be in this state. The `stack_limit` and `last_enter_wasm_sp` fields of
41863d482c8SFrank Emrich /// the corresponding `VMStackLimits` object contain valid data, while the
41963d482c8SFrank Emrich /// `last_exit_wasm_*` fields contain arbitrary values. There is only one
42063d482c8SFrank Emrich /// exception to this: Note that a continuation that has been created with
42163d482c8SFrank Emrich /// cont.new, but never been resumed so far, is considered "suspended".
42263d482c8SFrank Emrich /// However, its `last_enter_wasm_sp` field contains undefined data. This is
42363d482c8SFrank Emrich /// justified, because when resume-ing a continuation for the first time, a
42463d482c8SFrank Emrich /// native-to-wasm trampoline is called, which sets up the
42563d482c8SFrank Emrich /// `last_wasm_entry_sp` in the `VMStoreContext` with the correct value,
42663d482c8SFrank Emrich /// thus restoring the necessary invariant.
42763d482c8SFrank Emrich #[derive(Debug, Clone, PartialEq)]
42863d482c8SFrank Emrich #[repr(usize, C)]
42963d482c8SFrank Emrich pub enum VMStackChain {
43063d482c8SFrank Emrich /// For suspended continuations, denotes the end of their chain of
43163d482c8SFrank Emrich /// ancestors.
43263d482c8SFrank Emrich Absent = wasmtime_environ::STACK_CHAIN_ABSENT_DISCRIMINANT,
43363d482c8SFrank Emrich /// Represents the initial stack (i.e., where we entered Wasm from the
43463d482c8SFrank Emrich /// host by executing
43563d482c8SFrank Emrich /// `crate::runtime::func::invoke_wasm_and_catch_traps`). Therefore, it
43663d482c8SFrank Emrich /// does not have a parent. The `CommonStackInformation` that this
43763d482c8SFrank Emrich /// variant points to is stored in the stack frame of
43863d482c8SFrank Emrich /// `invoke_wasm_and_catch_traps`.
43963d482c8SFrank Emrich InitialStack(*mut VMCommonStackInformation) =
44063d482c8SFrank Emrich wasmtime_environ::STACK_CHAIN_INITIAL_STACK_DISCRIMINANT,
44163d482c8SFrank Emrich /// Represents a continuation's stack.
44263d482c8SFrank Emrich Continuation(*mut VMContRef) = wasmtime_environ::STACK_CHAIN_CONTINUATION_DISCRIMINANT,
44363d482c8SFrank Emrich }
44463d482c8SFrank Emrich
44563d482c8SFrank Emrich impl VMStackChain {
44663d482c8SFrank Emrich /// Indicates if `self` is a `InitialStack` variant.
is_initial_stack(&self) -> bool44763d482c8SFrank Emrich pub fn is_initial_stack(&self) -> bool {
44863d482c8SFrank Emrich matches!(self, VMStackChain::InitialStack(_))
44963d482c8SFrank Emrich }
45063d482c8SFrank Emrich
45163d482c8SFrank Emrich /// Returns an iterator over the continuations in this chain.
45263d482c8SFrank Emrich /// We don't implement `IntoIterator` because our iterator is unsafe, so at
45363d482c8SFrank Emrich /// least this gives us some way of indicating this, even though the actual
45463d482c8SFrank Emrich /// unsafety lies in the `next` function.
45563d482c8SFrank Emrich ///
45663d482c8SFrank Emrich /// # Safety
45763d482c8SFrank Emrich ///
45863d482c8SFrank Emrich /// This function is not unsafe per see, but it returns an object
45963d482c8SFrank Emrich /// whose usage is unsafe.
into_continuation_iter(self) -> ContinuationIterator46063d482c8SFrank Emrich pub unsafe fn into_continuation_iter(self) -> ContinuationIterator {
46163d482c8SFrank Emrich ContinuationIterator(self)
46263d482c8SFrank Emrich }
46363d482c8SFrank Emrich
46463d482c8SFrank Emrich /// Returns an iterator over the stack limits in this chain.
46563d482c8SFrank Emrich /// We don't implement `IntoIterator` because our iterator is unsafe, so at
46663d482c8SFrank Emrich /// least this gives us some way of indicating this, even though the actual
46763d482c8SFrank Emrich /// unsafety lies in the `next` function.
46863d482c8SFrank Emrich ///
46963d482c8SFrank Emrich /// # Safety
47063d482c8SFrank Emrich ///
47163d482c8SFrank Emrich /// This function is not unsafe per see, but it returns an object
47263d482c8SFrank Emrich /// whose usage is unsafe.
into_stack_limits_iter(self) -> StackLimitsIterator47363d482c8SFrank Emrich pub unsafe fn into_stack_limits_iter(self) -> StackLimitsIterator {
47463d482c8SFrank Emrich StackLimitsIterator(self)
47563d482c8SFrank Emrich }
47663d482c8SFrank Emrich }
47763d482c8SFrank Emrich
47863d482c8SFrank Emrich /// Iterator for Continuations in a stack chain.
47963d482c8SFrank Emrich pub struct ContinuationIterator(VMStackChain);
48063d482c8SFrank Emrich
48163d482c8SFrank Emrich /// Iterator for VMStackLimits in a stack chain.
48263d482c8SFrank Emrich pub struct StackLimitsIterator(VMStackChain);
48363d482c8SFrank Emrich
48463d482c8SFrank Emrich impl Iterator for ContinuationIterator {
48563d482c8SFrank Emrich type Item = *mut VMContRef;
48663d482c8SFrank Emrich
next(&mut self) -> Option<Self::Item>48763d482c8SFrank Emrich fn next(&mut self) -> Option<Self::Item> {
48863d482c8SFrank Emrich match self.0 {
48963d482c8SFrank Emrich VMStackChain::Absent | VMStackChain::InitialStack(_) => None,
49063d482c8SFrank Emrich VMStackChain::Continuation(ptr) => {
49163d482c8SFrank Emrich let continuation = unsafe { ptr.as_mut().unwrap() };
49263d482c8SFrank Emrich self.0 = continuation.parent_chain.clone();
49363d482c8SFrank Emrich Some(ptr)
49463d482c8SFrank Emrich }
49563d482c8SFrank Emrich }
49663d482c8SFrank Emrich }
49763d482c8SFrank Emrich }
49863d482c8SFrank Emrich
49963d482c8SFrank Emrich impl Iterator for StackLimitsIterator {
50063d482c8SFrank Emrich type Item = *mut VMStackLimits;
50163d482c8SFrank Emrich
next(&mut self) -> Option<Self::Item>50263d482c8SFrank Emrich fn next(&mut self) -> Option<Self::Item> {
50363d482c8SFrank Emrich match self.0 {
50463d482c8SFrank Emrich VMStackChain::Absent => None,
50563d482c8SFrank Emrich VMStackChain::InitialStack(csi) => {
50663d482c8SFrank Emrich let stack_limits = unsafe { &mut (*csi).limits } as *mut VMStackLimits;
50763d482c8SFrank Emrich self.0 = VMStackChain::Absent;
50863d482c8SFrank Emrich Some(stack_limits)
50963d482c8SFrank Emrich }
51063d482c8SFrank Emrich VMStackChain::Continuation(ptr) => {
51163d482c8SFrank Emrich let continuation = unsafe { ptr.as_mut().unwrap() };
51263d482c8SFrank Emrich let stack_limits =
51363d482c8SFrank Emrich (&mut continuation.common_stack_information.limits) as *mut VMStackLimits;
51463d482c8SFrank Emrich self.0 = continuation.parent_chain.clone();
51563d482c8SFrank Emrich Some(stack_limits)
51663d482c8SFrank Emrich }
51763d482c8SFrank Emrich }
51863d482c8SFrank Emrich }
51963d482c8SFrank Emrich }
52063d482c8SFrank Emrich
52163d482c8SFrank Emrich /// Encodes the life cycle of a `VMContRef`.
52263d482c8SFrank Emrich #[derive(Debug, Clone, Copy, PartialEq)]
52363d482c8SFrank Emrich #[repr(u32)]
52463d482c8SFrank Emrich pub enum VMStackState {
52563d482c8SFrank Emrich /// The `VMContRef` has been created, but neither `resume` or `switch` has ever been
52663d482c8SFrank Emrich /// called on it. During this stage, we may add arguments using `cont.bind`.
52763d482c8SFrank Emrich Fresh = wasmtime_environ::STACK_STATE_FRESH_DISCRIMINANT,
52863d482c8SFrank Emrich /// The continuation is running, meaning that it is the one currently
52963d482c8SFrank Emrich /// executing code.
53063d482c8SFrank Emrich Running = wasmtime_environ::STACK_STATE_RUNNING_DISCRIMINANT,
53163d482c8SFrank Emrich /// The continuation is suspended because it executed a resume instruction
53263d482c8SFrank Emrich /// that has not finished yet. In other words, it became the parent of
53363d482c8SFrank Emrich /// another continuation (which may itself be `Running`, a `Parent`, or
53463d482c8SFrank Emrich /// `Suspended`).
53563d482c8SFrank Emrich Parent = wasmtime_environ::STACK_STATE_PARENT_DISCRIMINANT,
53663d482c8SFrank Emrich /// The continuation was suspended by a `suspend` or `switch` instruction.
53763d482c8SFrank Emrich Suspended = wasmtime_environ::STACK_STATE_SUSPENDED_DISCRIMINANT,
53863d482c8SFrank Emrich /// The function originally passed to `cont.new` has returned normally.
53963d482c8SFrank Emrich /// Note that there is no guarantee that a VMContRef will ever
54063d482c8SFrank Emrich /// reach this status, as it may stay suspended until being dropped.
54163d482c8SFrank Emrich Returned = wasmtime_environ::STACK_STATE_RETURNED_DISCRIMINANT,
54263d482c8SFrank Emrich }
54363d482c8SFrank Emrich
54463d482c8SFrank Emrich #[cfg(test)]
54563d482c8SFrank Emrich mod tests {
54663d482c8SFrank Emrich use core::mem::{offset_of, size_of};
54763d482c8SFrank Emrich
5485245e1f8SNick Fitzgerald use wasmtime_environ::{HostPtr, Module, PtrSize, StaticModuleIndex, VMOffsets};
54963d482c8SFrank Emrich
55063d482c8SFrank Emrich use super::*;
55163d482c8SFrank Emrich
55263d482c8SFrank Emrich #[test]
null_pointer_optimization()55363d482c8SFrank Emrich fn null_pointer_optimization() {
55463d482c8SFrank Emrich // The Rust spec does not technically guarantee that the null pointer
55563d482c8SFrank Emrich // optimization applies to a struct containing a `NonNull`.
55663d482c8SFrank Emrich assert_eq!(size_of::<Option<VMContObj>>(), size_of::<VMContObj>());
55763d482c8SFrank Emrich }
55863d482c8SFrank Emrich
55963d482c8SFrank Emrich #[test]
check_vm_stack_limits_offsets()56063d482c8SFrank Emrich fn check_vm_stack_limits_offsets() {
5615245e1f8SNick Fitzgerald let module = Module::new(StaticModuleIndex::from_u32(0));
56263d482c8SFrank Emrich let offsets = VMOffsets::new(HostPtr, &module);
56363d482c8SFrank Emrich assert_eq!(
56463d482c8SFrank Emrich offset_of!(VMStackLimits, stack_limit),
56563d482c8SFrank Emrich usize::from(offsets.ptr.vmstack_limits_stack_limit())
56663d482c8SFrank Emrich );
56763d482c8SFrank Emrich assert_eq!(
56863d482c8SFrank Emrich offset_of!(VMStackLimits, last_wasm_entry_fp),
56963d482c8SFrank Emrich usize::from(offsets.ptr.vmstack_limits_last_wasm_entry_fp())
57063d482c8SFrank Emrich );
57163d482c8SFrank Emrich }
57263d482c8SFrank Emrich
57363d482c8SFrank Emrich #[test]
check_vm_common_stack_information_offsets()57463d482c8SFrank Emrich fn check_vm_common_stack_information_offsets() {
5755245e1f8SNick Fitzgerald let module = Module::new(StaticModuleIndex::from_u32(0));
57663d482c8SFrank Emrich let offsets = VMOffsets::new(HostPtr, &module);
57763d482c8SFrank Emrich assert_eq!(
57863d482c8SFrank Emrich size_of::<VMCommonStackInformation>(),
57963d482c8SFrank Emrich usize::from(offsets.ptr.size_of_vmcommon_stack_information())
58063d482c8SFrank Emrich );
58163d482c8SFrank Emrich assert_eq!(
58263d482c8SFrank Emrich offset_of!(VMCommonStackInformation, limits),
58363d482c8SFrank Emrich usize::from(offsets.ptr.vmcommon_stack_information_limits())
58463d482c8SFrank Emrich );
58563d482c8SFrank Emrich assert_eq!(
58663d482c8SFrank Emrich offset_of!(VMCommonStackInformation, state),
58763d482c8SFrank Emrich usize::from(offsets.ptr.vmcommon_stack_information_state())
58863d482c8SFrank Emrich );
58963d482c8SFrank Emrich assert_eq!(
59063d482c8SFrank Emrich offset_of!(VMCommonStackInformation, handlers),
59163d482c8SFrank Emrich usize::from(offsets.ptr.vmcommon_stack_information_handlers())
59263d482c8SFrank Emrich );
59363d482c8SFrank Emrich assert_eq!(
59463d482c8SFrank Emrich offset_of!(VMCommonStackInformation, first_switch_handler_index),
59563d482c8SFrank Emrich usize::from(
59663d482c8SFrank Emrich offsets
59763d482c8SFrank Emrich .ptr
59863d482c8SFrank Emrich .vmcommon_stack_information_first_switch_handler_index()
59963d482c8SFrank Emrich )
60063d482c8SFrank Emrich );
60163d482c8SFrank Emrich }
60263d482c8SFrank Emrich
60363d482c8SFrank Emrich #[test]
check_vm_array_offsets()60463d482c8SFrank Emrich fn check_vm_array_offsets() {
60563d482c8SFrank Emrich // Note that the type parameter has no influence on the size and offsets.
6065245e1f8SNick Fitzgerald let module = Module::new(StaticModuleIndex::from_u32(0));
60763d482c8SFrank Emrich let offsets = VMOffsets::new(HostPtr, &module);
60863d482c8SFrank Emrich assert_eq!(
60963d482c8SFrank Emrich size_of::<VMHostArray<()>>(),
6106ae11835SPaul Osborne usize::from(offsets.ptr.size_of_vmhostarray())
61163d482c8SFrank Emrich );
61263d482c8SFrank Emrich assert_eq!(
61363d482c8SFrank Emrich offset_of!(VMHostArray<()>, length),
6146ae11835SPaul Osborne usize::from(offsets.ptr.vmhostarray_length())
61563d482c8SFrank Emrich );
61663d482c8SFrank Emrich assert_eq!(
61763d482c8SFrank Emrich offset_of!(VMHostArray<()>, capacity),
6186ae11835SPaul Osborne usize::from(offsets.ptr.vmhostarray_capacity())
61963d482c8SFrank Emrich );
62063d482c8SFrank Emrich assert_eq!(
62163d482c8SFrank Emrich offset_of!(VMHostArray<()>, data),
6226ae11835SPaul Osborne usize::from(offsets.ptr.vmhostarray_data())
62363d482c8SFrank Emrich );
62463d482c8SFrank Emrich }
62563d482c8SFrank Emrich
62663d482c8SFrank Emrich #[test]
check_vm_contobj_offsets()627a631d20aSPaul Osborne fn check_vm_contobj_offsets() {
6285245e1f8SNick Fitzgerald let module = Module::new(StaticModuleIndex::from_u32(0));
629a631d20aSPaul Osborne let offsets = VMOffsets::new(HostPtr, &module);
630a631d20aSPaul Osborne assert_eq!(
631a631d20aSPaul Osborne offset_of!(VMContObj, contref),
632a631d20aSPaul Osborne usize::from(offsets.ptr.vmcontobj_contref())
633a631d20aSPaul Osborne );
634a631d20aSPaul Osborne assert_eq!(
635a631d20aSPaul Osborne offset_of!(VMContObj, revision),
636a631d20aSPaul Osborne usize::from(offsets.ptr.vmcontobj_revision())
637a631d20aSPaul Osborne );
638a631d20aSPaul Osborne assert_eq!(
639a631d20aSPaul Osborne size_of::<VMContObj>(),
640a631d20aSPaul Osborne usize::from(offsets.ptr.size_of_vmcontobj())
641a631d20aSPaul Osborne )
642a631d20aSPaul Osborne }
643a631d20aSPaul Osborne
644a631d20aSPaul Osborne #[test]
check_vm_contref_offsets()64563d482c8SFrank Emrich fn check_vm_contref_offsets() {
6465245e1f8SNick Fitzgerald let module = Module::new(StaticModuleIndex::from_u32(0));
64763d482c8SFrank Emrich let offsets = VMOffsets::new(HostPtr, &module);
64863d482c8SFrank Emrich assert_eq!(
64963d482c8SFrank Emrich offset_of!(VMContRef, common_stack_information),
65063d482c8SFrank Emrich usize::from(offsets.ptr.vmcontref_common_stack_information())
65163d482c8SFrank Emrich );
65263d482c8SFrank Emrich assert_eq!(
65363d482c8SFrank Emrich offset_of!(VMContRef, parent_chain),
65463d482c8SFrank Emrich usize::from(offsets.ptr.vmcontref_parent_chain())
65563d482c8SFrank Emrich );
65663d482c8SFrank Emrich assert_eq!(
65763d482c8SFrank Emrich offset_of!(VMContRef, last_ancestor),
65863d482c8SFrank Emrich usize::from(offsets.ptr.vmcontref_last_ancestor())
65963d482c8SFrank Emrich );
66063d482c8SFrank Emrich // Some 32-bit platforms need this to be 8-byte aligned, some don't.
66163d482c8SFrank Emrich // So we need to make sure it always is, without padding.
66263d482c8SFrank Emrich assert_eq!(u8::vmcontref_revision(&4) % 8, 0);
66363d482c8SFrank Emrich assert_eq!(u8::vmcontref_revision(&8) % 8, 0);
66463d482c8SFrank Emrich assert_eq!(
66563d482c8SFrank Emrich offset_of!(VMContRef, revision),
66663d482c8SFrank Emrich usize::from(offsets.ptr.vmcontref_revision())
66763d482c8SFrank Emrich );
66863d482c8SFrank Emrich assert_eq!(
66963d482c8SFrank Emrich offset_of!(VMContRef, stack),
67063d482c8SFrank Emrich usize::from(offsets.ptr.vmcontref_stack())
67163d482c8SFrank Emrich );
67263d482c8SFrank Emrich assert_eq!(
67363d482c8SFrank Emrich offset_of!(VMContRef, args),
67463d482c8SFrank Emrich usize::from(offsets.ptr.vmcontref_args())
67563d482c8SFrank Emrich );
67663d482c8SFrank Emrich assert_eq!(
67763d482c8SFrank Emrich offset_of!(VMContRef, values),
67863d482c8SFrank Emrich usize::from(offsets.ptr.vmcontref_values())
67963d482c8SFrank Emrich );
68063d482c8SFrank Emrich }
68163d482c8SFrank Emrich
68263d482c8SFrank Emrich #[test]
check_vm_stack_chain_offsets()68363d482c8SFrank Emrich fn check_vm_stack_chain_offsets() {
6845245e1f8SNick Fitzgerald let module = Module::new(StaticModuleIndex::from_u32(0));
68563d482c8SFrank Emrich let offsets = VMOffsets::new(HostPtr, &module);
68663d482c8SFrank Emrich assert_eq!(
68763d482c8SFrank Emrich size_of::<VMStackChain>(),
68863d482c8SFrank Emrich usize::from(offsets.ptr.size_of_vmstack_chain())
68963d482c8SFrank Emrich );
69063d482c8SFrank Emrich }
69163d482c8SFrank Emrich }
692