1 // A WORD OF CAUTION 2 // 3 // This entire file basically needs to be kept in sync with itself. It's not 4 // really possible to modify just one bit of this file without understanding 5 // all the other bits. Documentation tries to reference various bits here and 6 // there but try to make sure to read over everything before tweaking things! 7 // 8 // Also at this time this file is heavily based off the x86_64 file, so you'll 9 // probably want to read that one as well. 10 // 11 // Finally, control flow integrity hardening has been applied to the code using 12 // the Pointer Authentication (PAuth) and Branch Target Identification (BTI) 13 // technologies from the Arm instruction set architecture: 14 // * All callable functions start with either the `BTI c` or `PACIASP`/`PACIBSP` 15 // instructions 16 // * Return addresses are signed and authenticated using the stack pointer 17 // value as a modifier (similarly to the salt in a HMAC operation); the 18 // `DW_CFA_AARCH64_negate_ra_state` DWARF operation (aliased with the 19 // `.cfi_window_save` assembler directive) informs an unwinder about this 20 21 use core::arch::naked_asm; 22 23 cfg_if::cfg_if! { 24 if #[cfg(target_vendor = "apple")] { 25 macro_rules! paci1716 { () => ("pacib1716\n"); } 26 macro_rules! pacisp { () => ("pacibsp\n"); } 27 macro_rules! autisp { () => ("autibsp\n"); } 28 } else { 29 macro_rules! paci1716 { () => ("pacia1716\n"); } 30 macro_rules! pacisp { () => ("paciasp\n"); } 31 macro_rules! autisp { () => ("autiasp\n"); } 32 } 33 } 34 35 #[unsafe(naked)] 36 pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8 /* x0 */) { 37 naked_asm!(concat!( 38 " 39 .cfi_startproc 40 ", 41 pacisp!(), 42 " 43 .cfi_window_save 44 // Save all callee-saved registers on the stack since we're 45 // assuming they're clobbered as a result of the stack switch. 46 stp x29, x30, [sp, -16]! 47 stp x27, x28, [sp, -16]! 48 stp x25, x26, [sp, -16]! 49 stp x23, x24, [sp, -16]! 50 stp x21, x22, [sp, -16]! 51 stp x19, x20, [sp, -16]! 52 stp d14, d15, [sp, -16]! 53 stp d12, d13, [sp, -16]! 54 stp d10, d11, [sp, -16]! 55 stp d8, d9, [sp, -16]! 56 57 // Load our previously saved stack pointer to resume to, and save 58 // off our current stack pointer on where to come back to 59 // eventually. 60 ldr x8, [x0, -0x10] 61 mov x9, sp 62 str x9, [x0, -0x10] 63 64 // Switch to the new stack and restore all our callee-saved 65 // registers after the switch and return to our new stack. 66 mov sp, x8 67 ldp d8, d9, [sp], 16 68 ldp d10, d11, [sp], 16 69 ldp d12, d13, [sp], 16 70 ldp d14, d15, [sp], 16 71 72 ldp x19, x20, [sp], 16 73 ldp x21, x22, [sp], 16 74 ldp x23, x24, [sp], 16 75 ldp x25, x26, [sp], 16 76 ldp x27, x28, [sp], 16 77 ldp x29, x30, [sp], 16 78 ", 79 autisp!(), 80 " 81 .cfi_window_save 82 ret 83 .cfi_endproc 84 ", 85 )); 86 } 87 88 pub(crate) unsafe fn wasmtime_fiber_init( 89 top_of_stack: *mut u8, 90 entry_point: extern "C" fn(*mut u8, *mut u8), 91 entry_arg0: *mut u8, // x2 92 ) { 93 #[repr(C)] 94 #[derive(Default)] 95 struct InitialStack { 96 d8: u64, 97 d9: u64, 98 d10: u64, 99 d11: u64, 100 d12: u64, 101 d13: u64, 102 d14: u64, 103 d15: u64, 104 105 x19: *mut u8, 106 x20: *mut u8, 107 x21: *mut u8, 108 x22: *mut u8, 109 x23: *mut u8, 110 x24: *mut u8, 111 x25: *mut u8, 112 x26: *mut u8, 113 x27: *mut u8, 114 x28: *mut u8, 115 116 fp: *mut u8, 117 lr: *mut u8, 118 119 // unix.rs reserved space 120 last_sp: *mut u8, 121 run_result: *mut u8, 122 } 123 124 unsafe { 125 let initial_stack = top_of_stack.cast::<InitialStack>().sub(1); 126 initial_stack.write(InitialStack { 127 x19: top_of_stack, 128 x20: entry_point as *mut u8, 129 x21: entry_arg0, 130 131 // We set up the newly initialized fiber, so that it resumes 132 // execution from wasmtime_fiber_start(). As a result, we need a 133 // signed address of this function because `wasmtime_fiber_switch` 134 // ends with a `auti{a,b}sp` instruction. There are 2 requirements: 135 // * We would like to use an instruction that is executed as a no-op 136 // by processors that do not support PAuth, so that the code is 137 // backward-compatible and there is no duplication; `PACIA1716` is 138 // a suitable one. 139 // * The fiber stack pointer value that is used by the signing 140 // operation must match the value when the pointer is 141 // authenticated inside wasmtime_fiber_switch(), which is 16 bytes 142 // below the `top_of_stack` which will be `sp` at the time of the 143 // `auti{a,b}sp`. 144 // 145 // TODO: Use the PACGA instruction to authenticate the saved register 146 // state, which avoids creating signed pointers to 147 // wasmtime_fiber_start(), and provides wider coverage. 148 lr: paci1716(wasmtime_fiber_start as *mut u8, top_of_stack.sub(16)), 149 150 last_sp: initial_stack.cast(), 151 ..InitialStack::default() 152 }); 153 } 154 } 155 156 /// Signs `r17` with the value in `r16` using either `paci{a,b}1716` depending 157 /// on the platform. 158 fn paci1716(mut r17: *mut u8, r16: *mut u8) -> *mut u8 { 159 unsafe { 160 core::arch::asm!( 161 paci1716!(), 162 inout("x17") r17, 163 in("x16") r16, 164 ); 165 r17 166 } 167 } 168 169 // See the x86_64 file for more commentary on what these CFI directives are 170 // doing. Like over there note that the relative offsets to registers here 171 // match the frame layout in `wasmtime_fiber_switch`. 172 #[unsafe(naked)] 173 unsafe extern "C" fn wasmtime_fiber_start() -> ! { 174 naked_asm!( 175 " 176 .cfi_startproc simple 177 .cfi_def_cfa_offset 0 178 .cfi_escape 0x0f, /* DW_CFA_def_cfa_expression */ \ 179 5, /* the byte length of this expression */ \ 180 0x6f, /* DW_OP_reg31(%sp) */ \ 181 0x06, /* DW_OP_deref */ \ 182 0x23, 0xa0, 0x1 /* DW_OP_plus_uconst 0xa0 */ 183 .cfi_rel_offset x30, -0x08 184 .cfi_rel_offset x29, -0x10 185 .cfi_window_save 186 .cfi_rel_offset x28, -0x18 187 .cfi_rel_offset x27, -0x20 188 .cfi_rel_offset x26, -0x28 189 .cfi_rel_offset x25, -0x30 190 .cfi_rel_offset x24, -0x38 191 .cfi_rel_offset x23, -0x40 192 .cfi_rel_offset x22, -0x48 193 .cfi_rel_offset x21, -0x50 194 .cfi_rel_offset x20, -0x58 195 .cfi_rel_offset x19, -0x60 196 197 // Load our two arguments from the stack, where x1 is our start 198 // procedure and x0 is its first argument. This also blows away the 199 // stack space used by those two arguments. 200 mov x0, x21 201 mov x1, x19 202 203 // ... and then we call the function! Note that this is a function call 204 // so our frame stays on the stack to backtrace through. 205 blr x20 206 // Unreachable, here for safety. This should help catch unexpected 207 // behaviors. Use a noticeable payload so one can grep for it in the 208 // codebase. 209 brk 0xf1b3 210 .cfi_endproc 211 ", 212 ); 213 } 214