1 // A WORD OF CAUTION
2 //
3 // This entire file basically needs to be kept in sync with itself. It's not
4 // really possible to modify just one bit of this file without understanding
5 // all the other bits. Documentation tries to reference various bits here and
6 // there but try to make sure to read over everything before tweaking things!
7
8 use core::arch::naked_asm;
9
10 #[inline(never)] // FIXME(rust-lang/rust#148307)
wasmtime_fiber_switch(top_of_stack: *mut u8)11 pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8) {
12 unsafe { wasmtime_fiber_switch_(top_of_stack) }
13 }
14
15 #[unsafe(naked)]
wasmtime_fiber_switch_(top_of_stack: *mut u8 )16 unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* a0 */) {
17 naked_asm!(
18 "
19 // See https://github.com/rust-lang/rust/issues/80608.
20 .attribute arch, \"rv64gc\"
21
22 // We're switching to arbitrary code somewhere else, so pessimistically
23 // assume that all callee-save register are clobbered. This means we need
24 // to save/restore all of them.
25 //
26 // Note that this order for saving is important since we use CFI directives
27 // below to point to where all the saved registers are.
28 sd ra, -0x8(sp)
29 sd fp, -0x10(sp)
30 sd s1, -0x18(sp)
31 sd s2, -0x20(sp)
32 sd s3, -0x28(sp)
33 sd s4, -0x30(sp)
34 sd s5, -0x38(sp)
35 sd s6, -0x40(sp)
36 sd s7, -0x48(sp)
37 sd s8, -0x50(sp)
38 sd s9, -0x58(sp)
39 sd s10, -0x60(sp)
40 sd s11, -0x68(sp)
41 fsd fs0, -0x70(sp)
42 fsd fs1, -0x78(sp)
43 fsd fs2, -0x80(sp)
44 fsd fs3, -0x88(sp)
45 fsd fs4, -0x90(sp)
46 fsd fs5, -0x98(sp)
47 fsd fs6, -0xa0(sp)
48 fsd fs7, -0xa8(sp)
49 fsd fs8, -0xb0(sp)
50 fsd fs9, -0xb8(sp)
51 fsd fs10, -0xc0(sp)
52 fsd fs11, -0xc8(sp)
53 addi sp, sp, -0xd0
54
55 ld t0, -0x10(a0)
56 sd sp, -0x10(a0)
57
58 // Swap stacks and restore all our callee-saved registers
59 mv sp, t0
60
61 fld fs11, 0x8(sp)
62 fld fs10, 0x10(sp)
63 fld fs9, 0x18(sp)
64 fld fs8, 0x20(sp)
65 fld fs7, 0x28(sp)
66 fld fs6, 0x30(sp)
67 fld fs5, 0x38(sp)
68 fld fs4, 0x40(sp)
69 fld fs3, 0x48(sp)
70 fld fs2, 0x50(sp)
71 fld fs1, 0x58(sp)
72 fld fs0, 0x60(sp)
73 ld s11, 0x68(sp)
74 ld s10, 0x70(sp)
75 ld s9, 0x78(sp)
76 ld s8, 0x80(sp)
77 ld s7, 0x88(sp)
78 ld s6, 0x90(sp)
79 ld s5, 0x98(sp)
80 ld s4, 0xa0(sp)
81 ld s3, 0xa8(sp)
82 ld s2, 0xb0(sp)
83 ld s1, 0xb8(sp)
84 ld fp, 0xc0(sp)
85 ld ra, 0xc8(sp)
86 addi sp, sp, 0xd0
87 jr ra
88 ",
89 );
90 }
91
wasmtime_fiber_init( top_of_stack: *mut u8, entry_point: extern "C" fn(*mut u8, *mut u8) -> *mut u8, entry_arg0: *mut u8, )92 pub(crate) unsafe fn wasmtime_fiber_init(
93 top_of_stack: *mut u8,
94 entry_point: extern "C" fn(*mut u8, *mut u8) -> *mut u8,
95 entry_arg0: *mut u8,
96 ) {
97 #[repr(C)]
98 #[derive(Default)]
99 struct InitialStack {
100 align_to_16_byte_size: u64,
101
102 fs: [f64; 12],
103
104 s11: *mut u8,
105 s10: *mut u8,
106 s9: *mut u8,
107 s8: *mut u8,
108 s7: *mut u8,
109 s6: *mut u8,
110 s5: *mut u8,
111 s4: *mut u8,
112 s3: *mut u8,
113 s2: *mut u8,
114 s1: *mut u8,
115 fp: *mut u8,
116
117 ra: *mut u8,
118
119 // unix.rs reserved space
120 last_sp: *mut u8,
121 run_result: *mut u8,
122 }
123
124 unsafe {
125 let initial_stack = top_of_stack.cast::<InitialStack>().sub(1);
126 initial_stack.write(InitialStack {
127 s1: entry_point as *mut u8,
128 s2: entry_arg0,
129 s3: wasmtime_fiber_switch_ as *mut u8,
130 fp: top_of_stack,
131 ra: wasmtime_fiber_start as *mut u8,
132 last_sp: initial_stack.cast(),
133 ..InitialStack::default()
134 });
135 }
136 }
137
138 #[unsafe(naked)]
wasmtime_fiber_start() -> !139 unsafe extern "C" fn wasmtime_fiber_start() -> ! {
140 naked_asm!(
141 "
142 .cfi_startproc simple
143 .cfi_def_cfa_offset 0
144
145
146 .cfi_escape 0x0f, /* DW_CFA_def_cfa_expression */ \
147 5, /* the byte length of this expression */ \
148 0x52, /* DW_OP_reg2 (sp) */ \
149 0x06, /* DW_OP_deref */ \
150 0x08, 0xd0, /* DW_OP_const1u 0xc8 */ \
151 0x22 /* DW_OP_plus */
152
153
154 .cfi_rel_offset ra, -0x8
155 .cfi_rel_offset fp, -0x10
156 .cfi_rel_offset s1, -0x18
157 .cfi_rel_offset s2, -0x20
158 .cfi_rel_offset s3, -0x28
159 .cfi_rel_offset s4, -0x30
160 .cfi_rel_offset s5, -0x38
161 .cfi_rel_offset s6, -0x40
162 .cfi_rel_offset s7, -0x48
163 .cfi_rel_offset s8, -0x50
164 .cfi_rel_offset s9, -0x58
165 .cfi_rel_offset s10, -0x60
166 .cfi_rel_offset s11, -0x68
167 .cfi_rel_offset fs0, -0x70
168 .cfi_rel_offset fs1, -0x78
169 .cfi_rel_offset fs2, -0x80
170 .cfi_rel_offset fs3, -0x88
171 .cfi_rel_offset fs4, -0x90
172 .cfi_rel_offset fs5, -0x98
173 .cfi_rel_offset fs6, -0xa0
174 .cfi_rel_offset fs7, -0xa8
175 .cfi_rel_offset fs8, -0xb0
176 .cfi_rel_offset fs9, -0xb8
177 .cfi_rel_offset fs10, -0xc0
178 .cfi_rel_offset fs11, -0xc8
179
180 mv a0, s2
181 mv a1, fp
182 jalr s1 // entry_point
183 jalr s3 // wasmtime_fiber_switch_
184
185 // .4byte 0 will cause panic.
186 // for safety just like x86_64.rs.
187 .4byte 0
188 .cfi_endproc
189 ",
190 );
191 }
192