1 use super::regs; 2 use crate::{ 3 RegIndexEnv, Result, 4 abi::{ABI, ABIOperand, ABIParams, ABIResults, ABISig, ParamsOrReturns, align_to}, 5 bail, 6 codegen::CodeGenError, 7 isa::{CallingConvention, reg::Reg}, 8 }; 9 use wasmtime_environ::{WasmHeapType, WasmValType}; 10 11 #[derive(Default)] 12 pub(crate) struct X64ABI; 13 14 impl ABI for X64ABI { stack_align() -> u815 fn stack_align() -> u8 { 16 16 17 } 18 call_stack_align() -> u819 fn call_stack_align() -> u8 { 20 16 21 } 22 arg_base_offset() -> u823 fn arg_base_offset() -> u8 { 24 // Two 8-byte slots, one for the return address and another 25 // one for the frame pointer. 26 // ┌──────────┬───────── Argument base 27 // │ Ret │ 28 // │ Addr │ 29 // ├──────────┼ 30 // │ │ 31 // │ FP │ 32 // └──────────┴ 33 16 34 } 35 initial_frame_size() -> u836 fn initial_frame_size() -> u8 { 37 // The initial frame size is equal to the space allocated to save the 38 // return address and the frame pointer. 39 Self::arg_base_offset() 40 } 41 word_bits() -> u842 fn word_bits() -> u8 { 43 64 44 } 45 sig_from( params: &[WasmValType], returns: &[WasmValType], call_conv: &CallingConvention, ) -> Result<ABISig>46 fn sig_from( 47 params: &[WasmValType], 48 returns: &[WasmValType], 49 call_conv: &CallingConvention, 50 ) -> Result<ABISig> { 51 assert!(call_conv.is_fastcall() || call_conv.is_systemv() || call_conv.is_default()); 52 let is_fastcall = call_conv.is_fastcall(); 53 // In the fastcall calling convention, the callee gets a contiguous 54 // stack area of 32 bytes (4 register arguments) just before its frame. 55 // See 56 // https://learn.microsoft.com/en-us/cpp/build/stack-usage?view=msvc-170#stack-allocation 57 let (params_stack_offset, mut params_index_env) = if is_fastcall { 58 (32, RegIndexEnv::with_absolute_limit(4)) 59 } else { 60 (0, RegIndexEnv::with_limits_per_class(6, 8)) 61 }; 62 63 let results = Self::abi_results(returns, call_conv)?; 64 let params = ABIParams::from::<_, Self>( 65 params, 66 params_stack_offset, 67 results.on_stack(), 68 |ty, stack_offset| { 69 Self::to_abi_operand( 70 ty, 71 stack_offset, 72 &mut params_index_env, 73 call_conv, 74 ParamsOrReturns::Params, 75 ) 76 }, 77 )?; 78 79 Ok(ABISig::new(*call_conv, params, results)) 80 } 81 abi_results(returns: &[WasmValType], call_conv: &CallingConvention) -> Result<ABIResults>82 fn abi_results(returns: &[WasmValType], call_conv: &CallingConvention) -> Result<ABIResults> { 83 assert!(call_conv.is_default() || call_conv.is_fastcall() || call_conv.is_systemv()); 84 // Use absolute count for results given that for Winch's 85 // default CallingConvention only one register is used for results 86 // independent of the register class. 87 // In the case of 2+ results, the rest are passed in the stack, 88 // similar to how Wasmtime handles multi-value returns. 89 let mut results_index_env = RegIndexEnv::with_absolute_limit(1); 90 ABIResults::from(returns, call_conv, |ty, offset| { 91 Self::to_abi_operand( 92 ty, 93 offset, 94 &mut results_index_env, 95 call_conv, 96 ParamsOrReturns::Returns, 97 ) 98 }) 99 } 100 vmctx_reg() -> Reg101 fn vmctx_reg() -> Reg { 102 regs::vmctx() 103 } 104 stack_slot_size() -> u8105 fn stack_slot_size() -> u8 { 106 // Winch default calling convention follows SysV calling convention so 107 // we use one 8 byte slot for values that are smaller or equal to 8 108 // bytes in size and 2 8 byte slots for values that are 128 bits. 109 // See Section 3.2.3 in 110 // https://refspecs.linuxbase.org/elf/x86_64-abi-0.99.pdf for further 111 // details. 112 Self::word_bytes() 113 } 114 sizeof(ty: &WasmValType) -> u8115 fn sizeof(ty: &WasmValType) -> u8 { 116 match ty { 117 WasmValType::Ref(rt) => match rt.heap_type { 118 WasmHeapType::Func | WasmHeapType::Extern => Self::word_bytes(), 119 ht => unimplemented!("Support for WasmHeapType: {ht}"), 120 }, 121 WasmValType::F64 | WasmValType::I64 => Self::word_bytes(), 122 WasmValType::F32 | WasmValType::I32 => Self::word_bytes() / 2, 123 WasmValType::V128 => Self::word_bytes() * 2, 124 } 125 } 126 } 127 128 impl X64ABI { to_abi_operand( wasm_arg: &WasmValType, stack_offset: u32, index_env: &mut RegIndexEnv, call_conv: &CallingConvention, params_or_returns: ParamsOrReturns, ) -> Result<(ABIOperand, u32)>129 fn to_abi_operand( 130 wasm_arg: &WasmValType, 131 stack_offset: u32, 132 index_env: &mut RegIndexEnv, 133 call_conv: &CallingConvention, 134 params_or_returns: ParamsOrReturns, 135 ) -> Result<(ABIOperand, u32)> { 136 let (reg, ty) = match wasm_arg { 137 ty @ WasmValType::Ref(rt) => match rt.heap_type { 138 WasmHeapType::Func | WasmHeapType::Extern => ( 139 Self::int_reg_for(index_env.next_gpr(), call_conv, params_or_returns), 140 ty, 141 ), 142 _ => bail!(CodeGenError::unsupported_wasm_type()), 143 }, 144 145 ty @ (WasmValType::I32 | WasmValType::I64) => ( 146 Self::int_reg_for(index_env.next_gpr(), call_conv, params_or_returns), 147 ty, 148 ), 149 150 // v128 also uses an XMM register (that is, an fpr). 151 ty @ (WasmValType::F32 | WasmValType::F64 | WasmValType::V128) => ( 152 Self::float_reg_for(index_env.next_fpr(), call_conv, params_or_returns), 153 ty, 154 ), 155 }; 156 157 let ty_size = <Self as ABI>::sizeof(wasm_arg); 158 let default = || { 159 let slot_size = Self::stack_slot_size(); 160 if params_or_returns == ParamsOrReturns::Params { 161 // Stack slots for parameters are aligned to a fixed slot size, 162 // 8 bytes if the type size is 8 or less and type-sized aligned 163 // if the type size is greater than 8 bytes. 164 let alignment = std::cmp::max(ty_size, slot_size); 165 let offset = align_to(stack_offset, u32::from(alignment)); 166 let arg = ABIOperand::stack_offset(offset, *ty, u32::from(ty_size)); 167 (arg, offset + u32::from(alignment)) 168 } else { 169 // For the default calling convention, we don't type-size align, 170 // given that results on the stack must match spills generated 171 // from within the compiler, which are not type-size aligned. 172 // In all other cases the results are type-sized aligned. 173 if call_conv.is_default() { 174 let arg = ABIOperand::stack_offset(stack_offset, *ty, u32::from(ty_size)); 175 (arg, stack_offset + (ty_size as u32)) 176 } else { 177 let offset = align_to(stack_offset, u32::from(ty_size)); 178 ( 179 ABIOperand::stack_offset(offset, *ty, u32::from(ty_size)), 180 offset + u32::from(ty_size), 181 ) 182 } 183 } 184 }; 185 186 Ok(reg.map_or_else(default, |reg| { 187 (ABIOperand::reg(reg, *ty, ty_size as u32), stack_offset) 188 })) 189 } 190 int_reg_for( index: Option<u8>, call_conv: &CallingConvention, params_or_returns: ParamsOrReturns, ) -> Option<Reg>191 fn int_reg_for( 192 index: Option<u8>, 193 call_conv: &CallingConvention, 194 params_or_returns: ParamsOrReturns, 195 ) -> Option<Reg> { 196 use ParamsOrReturns::*; 197 198 let index = match index { 199 None => return None, 200 Some(index) => index, 201 }; 202 203 if call_conv.is_fastcall() { 204 return match (index, params_or_returns) { 205 (0, Params) => Some(regs::rcx()), 206 (1, Params) => Some(regs::rdx()), 207 (2, Params) => Some(regs::r8()), 208 (3, Params) => Some(regs::r9()), 209 (0, Returns) => Some(regs::rax()), 210 _ => None, 211 }; 212 } 213 214 if call_conv.is_systemv() || call_conv.is_default() { 215 return match (index, params_or_returns) { 216 (0, Params) => Some(regs::rdi()), 217 (1, Params) => Some(regs::rsi()), 218 (2, Params) => Some(regs::rdx()), 219 (3, Params) => Some(regs::rcx()), 220 (4, Params) => Some(regs::r8()), 221 (5, Params) => Some(regs::r9()), 222 (0, Returns) => Some(regs::rax()), 223 _ => None, 224 }; 225 } 226 227 None 228 } 229 float_reg_for( index: Option<u8>, call_conv: &CallingConvention, params_or_returns: ParamsOrReturns, ) -> Option<Reg>230 fn float_reg_for( 231 index: Option<u8>, 232 call_conv: &CallingConvention, 233 params_or_returns: ParamsOrReturns, 234 ) -> Option<Reg> { 235 use ParamsOrReturns::*; 236 237 let index = match index { 238 None => return None, 239 Some(index) => index, 240 }; 241 242 if call_conv.is_fastcall() { 243 return match (index, params_or_returns) { 244 (0, Params) => Some(regs::xmm0()), 245 (1, Params) => Some(regs::xmm1()), 246 (2, Params) => Some(regs::xmm2()), 247 (3, Params) => Some(regs::xmm3()), 248 (0, Returns) => Some(regs::xmm0()), 249 _ => None, 250 }; 251 } 252 253 if call_conv.is_systemv() || call_conv.is_default() { 254 return match (index, params_or_returns) { 255 (0, Params) => Some(regs::xmm0()), 256 (1, Params) => Some(regs::xmm1()), 257 (2, Params) => Some(regs::xmm2()), 258 (3, Params) => Some(regs::xmm3()), 259 (4, Params) => Some(regs::xmm4()), 260 (5, Params) => Some(regs::xmm5()), 261 (6, Params) => Some(regs::xmm6()), 262 (7, Params) => Some(regs::xmm7()), 263 (0, Returns) => Some(regs::xmm0()), 264 _ => None, 265 }; 266 } 267 268 None 269 } 270 } 271 272 #[cfg(test)] 273 mod tests { 274 use super::X64ABI; 275 use crate::{ 276 Result, 277 abi::{ABI, ABIOperand}, 278 isa::{CallingConvention, reg::Reg, x64::regs}, 279 }; 280 use wasmtime_environ::{ 281 WasmFuncType, 282 WasmValType::{self, *}, 283 }; 284 285 #[test] int_abi_sig() -> Result<()>286 fn int_abi_sig() -> Result<()> { 287 let wasm_sig = WasmFuncType::new([I32, I64, I32, I64, I32, I32, I64, I32], [])?; 288 289 let sig = X64ABI::sig(&wasm_sig, &CallingConvention::Default)?; 290 let params = sig.params; 291 292 match_reg_arg(params.get(0).unwrap(), I32, regs::rdi()); 293 match_reg_arg(params.get(1).unwrap(), I64, regs::rsi()); 294 match_reg_arg(params.get(2).unwrap(), I32, regs::rdx()); 295 match_reg_arg(params.get(3).unwrap(), I64, regs::rcx()); 296 match_reg_arg(params.get(4).unwrap(), I32, regs::r8()); 297 match_reg_arg(params.get(5).unwrap(), I32, regs::r9()); 298 match_stack_arg(params.get(6).unwrap(), I64, 0); 299 match_stack_arg(params.get(7).unwrap(), I32, 8); 300 Ok(()) 301 } 302 303 #[test] int_abi_sig_multi_returns() -> Result<()>304 fn int_abi_sig_multi_returns() -> Result<()> { 305 let wasm_sig = 306 WasmFuncType::new([I32, I64, I32, I64, I32, I32, I64, I32], [I32, I32, I32])?; 307 308 let sig = X64ABI::sig(&wasm_sig, &CallingConvention::Default)?; 309 let params = sig.params; 310 let results = sig.results; 311 312 match_reg_arg(params.get(0).unwrap(), I32, regs::rsi()); 313 match_reg_arg(params.get(1).unwrap(), I64, regs::rdx()); 314 match_reg_arg(params.get(2).unwrap(), I32, regs::rcx()); 315 match_reg_arg(params.get(3).unwrap(), I64, regs::r8()); 316 match_reg_arg(params.get(4).unwrap(), I32, regs::r9()); 317 match_stack_arg(params.get(5).unwrap(), I32, 0); 318 match_stack_arg(params.get(6).unwrap(), I64, 8); 319 match_stack_arg(params.get(7).unwrap(), I32, 16); 320 321 match_stack_arg(results.get(0).unwrap(), I32, 4); 322 match_stack_arg(results.get(1).unwrap(), I32, 0); 323 match_reg_arg(results.get(2).unwrap(), I32, regs::rax()); 324 Ok(()) 325 } 326 327 #[test] float_abi_sig() -> Result<()>328 fn float_abi_sig() -> Result<()> { 329 let wasm_sig = WasmFuncType::new([F32, F64, F32, F64, F32, F32, F64, F32, F64], [])?; 330 331 let sig = X64ABI::sig(&wasm_sig, &CallingConvention::Default)?; 332 let params = sig.params; 333 334 match_reg_arg(params.get(0).unwrap(), F32, regs::xmm0()); 335 match_reg_arg(params.get(1).unwrap(), F64, regs::xmm1()); 336 match_reg_arg(params.get(2).unwrap(), F32, regs::xmm2()); 337 match_reg_arg(params.get(3).unwrap(), F64, regs::xmm3()); 338 match_reg_arg(params.get(4).unwrap(), F32, regs::xmm4()); 339 match_reg_arg(params.get(5).unwrap(), F32, regs::xmm5()); 340 match_reg_arg(params.get(6).unwrap(), F64, regs::xmm6()); 341 match_reg_arg(params.get(7).unwrap(), F32, regs::xmm7()); 342 match_stack_arg(params.get(8).unwrap(), F64, 0); 343 Ok(()) 344 } 345 346 #[test] vector_abi_sig() -> Result<()>347 fn vector_abi_sig() -> Result<()> { 348 let wasm_sig = WasmFuncType::new( 349 [V128, V128, V128, V128, V128, V128, V128, V128, V128, V128], 350 [], 351 )?; 352 353 let sig = X64ABI::sig(&wasm_sig, &CallingConvention::Default)?; 354 let params = sig.params; 355 356 match_reg_arg(params.get(0).unwrap(), V128, regs::xmm0()); 357 match_reg_arg(params.get(1).unwrap(), V128, regs::xmm1()); 358 match_reg_arg(params.get(2).unwrap(), V128, regs::xmm2()); 359 match_reg_arg(params.get(3).unwrap(), V128, regs::xmm3()); 360 match_reg_arg(params.get(4).unwrap(), V128, regs::xmm4()); 361 match_reg_arg(params.get(5).unwrap(), V128, regs::xmm5()); 362 match_reg_arg(params.get(6).unwrap(), V128, regs::xmm6()); 363 match_reg_arg(params.get(7).unwrap(), V128, regs::xmm7()); 364 match_stack_arg(params.get(8).unwrap(), V128, 0); 365 match_stack_arg(params.get(9).unwrap(), V128, 16); 366 Ok(()) 367 } 368 369 #[test] vector_abi_sig_multi_returns() -> Result<()>370 fn vector_abi_sig_multi_returns() -> Result<()> { 371 let wasm_sig = WasmFuncType::new([], [V128, V128, V128])?; 372 373 let sig = X64ABI::sig(&wasm_sig, &CallingConvention::Default)?; 374 let results = sig.results; 375 376 match_stack_arg(results.get(0).unwrap(), V128, 16); 377 match_stack_arg(results.get(1).unwrap(), V128, 0); 378 match_reg_arg(results.get(2).unwrap(), V128, regs::xmm0()); 379 Ok(()) 380 } 381 382 #[test] mixed_abi_sig() -> Result<()>383 fn mixed_abi_sig() -> Result<()> { 384 let wasm_sig = WasmFuncType::new([F32, I32, I64, F64, I32, F32, F64, F32, F64], [])?; 385 386 let sig = X64ABI::sig(&wasm_sig, &CallingConvention::Default)?; 387 let params = sig.params; 388 389 match_reg_arg(params.get(0).unwrap(), F32, regs::xmm0()); 390 match_reg_arg(params.get(1).unwrap(), I32, regs::rdi()); 391 match_reg_arg(params.get(2).unwrap(), I64, regs::rsi()); 392 match_reg_arg(params.get(3).unwrap(), F64, regs::xmm1()); 393 match_reg_arg(params.get(4).unwrap(), I32, regs::rdx()); 394 match_reg_arg(params.get(5).unwrap(), F32, regs::xmm2()); 395 match_reg_arg(params.get(6).unwrap(), F64, regs::xmm3()); 396 match_reg_arg(params.get(7).unwrap(), F32, regs::xmm4()); 397 match_reg_arg(params.get(8).unwrap(), F64, regs::xmm5()); 398 399 let wasm_sig = 400 WasmFuncType::new([F32, F32, F32, F32, F32, F32, F32, F32, F32, V128], [V128])?; 401 402 let sig = X64ABI::sig(&wasm_sig, &CallingConvention::Default)?; 403 let params = sig.params; 404 405 match_stack_arg(params.get(8).unwrap(), F32, 0); 406 match_stack_arg(params.get(9).unwrap(), V128, 16); 407 Ok(()) 408 } 409 410 #[test] system_v_call_conv() -> Result<()>411 fn system_v_call_conv() -> Result<()> { 412 let wasm_sig = WasmFuncType::new([F32, I32, I64, F64, I32, F32, F64, F32, F64], [])?; 413 414 let sig = X64ABI::sig(&wasm_sig, &CallingConvention::SystemV)?; 415 let params = sig.params; 416 417 match_reg_arg(params.get(0).unwrap(), F32, regs::xmm0()); 418 match_reg_arg(params.get(1).unwrap(), I32, regs::rdi()); 419 match_reg_arg(params.get(2).unwrap(), I64, regs::rsi()); 420 match_reg_arg(params.get(3).unwrap(), F64, regs::xmm1()); 421 match_reg_arg(params.get(4).unwrap(), I32, regs::rdx()); 422 match_reg_arg(params.get(5).unwrap(), F32, regs::xmm2()); 423 match_reg_arg(params.get(6).unwrap(), F64, regs::xmm3()); 424 match_reg_arg(params.get(7).unwrap(), F32, regs::xmm4()); 425 match_reg_arg(params.get(8).unwrap(), F64, regs::xmm5()); 426 Ok(()) 427 } 428 429 #[test] fastcall_call_conv() -> Result<()>430 fn fastcall_call_conv() -> Result<()> { 431 let wasm_sig = WasmFuncType::new([F32, I32, I64, F64, I32, F32, F64, F32, F64], [])?; 432 433 let sig = X64ABI::sig(&wasm_sig, &CallingConvention::WindowsFastcall)?; 434 let params = sig.params; 435 436 match_reg_arg(params.get(0).unwrap(), F32, regs::xmm0()); 437 match_reg_arg(params.get(1).unwrap(), I32, regs::rdx()); 438 match_reg_arg(params.get(2).unwrap(), I64, regs::r8()); 439 match_reg_arg(params.get(3).unwrap(), F64, regs::xmm3()); 440 match_stack_arg(params.get(4).unwrap(), I32, 32); 441 match_stack_arg(params.get(5).unwrap(), F32, 40); 442 Ok(()) 443 } 444 445 #[test] fastcall_call_conv_multi_returns() -> Result<()>446 fn fastcall_call_conv_multi_returns() -> Result<()> { 447 let wasm_sig = WasmFuncType::new( 448 [F32, I32, I64, F64, I32, F32, F64, F32, F64], 449 [I32, F32, I32, F32, I64], 450 )?; 451 452 let sig = X64ABI::sig(&wasm_sig, &CallingConvention::WindowsFastcall)?; 453 let params = sig.params; 454 let results = sig.results; 455 456 match_reg_arg(params.get(0).unwrap(), F32, regs::xmm1()); 457 match_reg_arg(params.get(1).unwrap(), I32, regs::r8()); 458 match_reg_arg(params.get(2).unwrap(), I64, regs::r9()); 459 // Each argument stack slot is 8 bytes. 460 match_stack_arg(params.get(3).unwrap(), F64, 32); 461 match_stack_arg(params.get(4).unwrap(), I32, 40); 462 match_stack_arg(params.get(5).unwrap(), F32, 48); 463 464 match_reg_arg(results.get(0).unwrap(), I32, regs::rax()); 465 466 match_stack_arg(results.get(1).unwrap(), F32, 0); 467 match_stack_arg(results.get(2).unwrap(), I32, 4); 468 match_stack_arg(results.get(3).unwrap(), F32, 8); 469 match_stack_arg(results.get(4).unwrap(), I64, 16); 470 Ok(()) 471 } 472 473 #[track_caller] match_reg_arg(abi_arg: &ABIOperand, expected_ty: WasmValType, expected_reg: Reg)474 fn match_reg_arg(abi_arg: &ABIOperand, expected_ty: WasmValType, expected_reg: Reg) { 475 match abi_arg { 476 &ABIOperand::Reg { reg, ty, .. } => { 477 assert_eq!(reg, expected_reg); 478 assert_eq!(ty, expected_ty); 479 } 480 stack => panic!("Expected reg argument, got {stack:?}"), 481 } 482 } 483 484 #[track_caller] match_stack_arg(abi_arg: &ABIOperand, expected_ty: WasmValType, expected_offset: u32)485 fn match_stack_arg(abi_arg: &ABIOperand, expected_ty: WasmValType, expected_offset: u32) { 486 match abi_arg { 487 &ABIOperand::Stack { offset, ty, .. } => { 488 assert_eq!(offset, expected_offset); 489 assert_eq!(ty, expected_ty); 490 } 491 reg => panic!("Expected stack argument, got {reg:?}"), 492 } 493 } 494 } 495