1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2018, Google LLC. 4 */ 5 #ifndef SELFTEST_KVM_UTIL_H 6 #define SELFTEST_KVM_UTIL_H 7 8 #include "test_util.h" 9 10 #include <linux/compiler.h> 11 #include "linux/hashtable.h" 12 #include "linux/list.h" 13 #include <linux/kernel.h> 14 #include <linux/kvm.h> 15 #include "linux/rbtree.h" 16 #include <linux/types.h> 17 18 #include <asm/atomic.h> 19 #include <asm/kvm.h> 20 21 #include <sys/ioctl.h> 22 23 #include "kvm_util_arch.h" 24 #include "kvm_util_types.h" 25 #include "sparsebit.h" 26 27 #define KVM_DEV_PATH "/dev/kvm" 28 #define KVM_MAX_VCPUS 512 29 30 #define NSEC_PER_SEC 1000000000L 31 32 struct userspace_mem_region { 33 struct kvm_userspace_memory_region2 region; 34 struct sparsebit *unused_phy_pages; 35 struct sparsebit *protected_phy_pages; 36 int fd; 37 off_t offset; 38 enum vm_mem_backing_src_type backing_src_type; 39 void *host_mem; 40 void *host_alias; 41 void *mmap_start; 42 void *mmap_alias; 43 size_t mmap_size; 44 struct rb_node gpa_node; 45 struct rb_node hva_node; 46 struct hlist_node slot_node; 47 }; 48 49 struct kvm_vcpu { 50 struct list_head list; 51 uint32_t id; 52 int fd; 53 struct kvm_vm *vm; 54 struct kvm_run *run; 55 #ifdef __x86_64__ 56 struct kvm_cpuid2 *cpuid; 57 #endif 58 struct kvm_dirty_gfn *dirty_gfns; 59 uint32_t fetch_index; 60 uint32_t dirty_gfns_count; 61 }; 62 63 struct userspace_mem_regions { 64 struct rb_root gpa_tree; 65 struct rb_root hva_tree; 66 DECLARE_HASHTABLE(slot_hash, 9); 67 }; 68 69 enum kvm_mem_region_type { 70 MEM_REGION_CODE, 71 MEM_REGION_DATA, 72 MEM_REGION_PT, 73 MEM_REGION_TEST_DATA, 74 NR_MEM_REGIONS, 75 }; 76 77 struct kvm_vm { 78 int mode; 79 unsigned long type; 80 int kvm_fd; 81 int fd; 82 unsigned int pgtable_levels; 83 unsigned int page_size; 84 unsigned int page_shift; 85 unsigned int pa_bits; 86 unsigned int va_bits; 87 uint64_t max_gfn; 88 struct list_head vcpus; 89 struct userspace_mem_regions regions; 90 struct sparsebit *vpages_valid; 91 struct sparsebit *vpages_mapped; 92 bool has_irqchip; 93 bool pgd_created; 94 vm_paddr_t ucall_mmio_addr; 95 vm_paddr_t pgd; 96 vm_vaddr_t gdt; 97 vm_vaddr_t tss; 98 vm_vaddr_t idt; 99 vm_vaddr_t handlers; 100 uint32_t dirty_ring_size; 101 uint64_t gpa_tag_mask; 102 103 struct kvm_vm_arch arch; 104 105 /* Cache of information for binary stats interface */ 106 int stats_fd; 107 struct kvm_stats_header stats_header; 108 struct kvm_stats_desc *stats_desc; 109 110 /* 111 * KVM region slots. These are the default memslots used by page 112 * allocators, e.g., lib/elf uses the memslots[MEM_REGION_CODE] 113 * memslot. 114 */ 115 uint32_t memslots[NR_MEM_REGIONS]; 116 }; 117 118 struct vcpu_reg_sublist { 119 const char *name; 120 long capability; 121 int feature; 122 int feature_type; 123 bool finalize; 124 __u64 *regs; 125 __u64 regs_n; 126 __u64 *rejects_set; 127 __u64 rejects_set_n; 128 __u64 *skips_set; 129 __u64 skips_set_n; 130 }; 131 132 struct vcpu_reg_list { 133 char *name; 134 struct vcpu_reg_sublist sublists[]; 135 }; 136 137 #define for_each_sublist(c, s) \ 138 for ((s) = &(c)->sublists[0]; (s)->regs; ++(s)) 139 140 #define kvm_for_each_vcpu(vm, i, vcpu) \ 141 for ((i) = 0; (i) <= (vm)->last_vcpu_id; (i)++) \ 142 if (!((vcpu) = vm->vcpus[i])) \ 143 continue; \ 144 else 145 146 struct userspace_mem_region * 147 memslot2region(struct kvm_vm *vm, uint32_t memslot); 148 149 static inline struct userspace_mem_region *vm_get_mem_region(struct kvm_vm *vm, 150 enum kvm_mem_region_type type) 151 { 152 assert(type < NR_MEM_REGIONS); 153 return memslot2region(vm, vm->memslots[type]); 154 } 155 156 /* Minimum allocated guest virtual and physical addresses */ 157 #define KVM_UTIL_MIN_VADDR 0x2000 158 #define KVM_GUEST_PAGE_TABLE_MIN_PADDR 0x180000 159 160 #define DEFAULT_GUEST_STACK_VADDR_MIN 0xab6000 161 #define DEFAULT_STACK_PGS 5 162 163 enum vm_guest_mode { 164 VM_MODE_P52V48_4K, 165 VM_MODE_P52V48_16K, 166 VM_MODE_P52V48_64K, 167 VM_MODE_P48V48_4K, 168 VM_MODE_P48V48_16K, 169 VM_MODE_P48V48_64K, 170 VM_MODE_P40V48_4K, 171 VM_MODE_P40V48_16K, 172 VM_MODE_P40V48_64K, 173 VM_MODE_PXXV48_4K, /* For 48bits VA but ANY bits PA */ 174 VM_MODE_P47V64_4K, 175 VM_MODE_P44V64_4K, 176 VM_MODE_P36V48_4K, 177 VM_MODE_P36V48_16K, 178 VM_MODE_P36V48_64K, 179 VM_MODE_P36V47_16K, 180 NUM_VM_MODES, 181 }; 182 183 struct vm_shape { 184 uint32_t type; 185 uint8_t mode; 186 uint8_t pad0; 187 uint16_t pad1; 188 }; 189 190 kvm_static_assert(sizeof(struct vm_shape) == sizeof(uint64_t)); 191 192 #define VM_TYPE_DEFAULT 0 193 194 #define VM_SHAPE(__mode) \ 195 ({ \ 196 struct vm_shape shape = { \ 197 .mode = (__mode), \ 198 .type = VM_TYPE_DEFAULT \ 199 }; \ 200 \ 201 shape; \ 202 }) 203 204 #if defined(__aarch64__) 205 206 extern enum vm_guest_mode vm_mode_default; 207 208 #define VM_MODE_DEFAULT vm_mode_default 209 #define MIN_PAGE_SHIFT 12U 210 #define ptes_per_page(page_size) ((page_size) / 8) 211 212 #elif defined(__x86_64__) 213 214 #define VM_MODE_DEFAULT VM_MODE_PXXV48_4K 215 #define MIN_PAGE_SHIFT 12U 216 #define ptes_per_page(page_size) ((page_size) / 8) 217 218 #elif defined(__s390x__) 219 220 #define VM_MODE_DEFAULT VM_MODE_P44V64_4K 221 #define MIN_PAGE_SHIFT 12U 222 #define ptes_per_page(page_size) ((page_size) / 16) 223 224 #elif defined(__riscv) 225 226 #if __riscv_xlen == 32 227 #error "RISC-V 32-bit kvm selftests not supported" 228 #endif 229 230 #define VM_MODE_DEFAULT VM_MODE_P40V48_4K 231 #define MIN_PAGE_SHIFT 12U 232 #define ptes_per_page(page_size) ((page_size) / 8) 233 234 #endif 235 236 #define VM_SHAPE_DEFAULT VM_SHAPE(VM_MODE_DEFAULT) 237 238 #define MIN_PAGE_SIZE (1U << MIN_PAGE_SHIFT) 239 #define PTES_PER_MIN_PAGE ptes_per_page(MIN_PAGE_SIZE) 240 241 struct vm_guest_mode_params { 242 unsigned int pa_bits; 243 unsigned int va_bits; 244 unsigned int page_size; 245 unsigned int page_shift; 246 }; 247 extern const struct vm_guest_mode_params vm_guest_mode_params[]; 248 249 int open_path_or_exit(const char *path, int flags); 250 int open_kvm_dev_path_or_exit(void); 251 252 bool get_kvm_param_bool(const char *param); 253 bool get_kvm_intel_param_bool(const char *param); 254 bool get_kvm_amd_param_bool(const char *param); 255 256 int get_kvm_param_integer(const char *param); 257 int get_kvm_intel_param_integer(const char *param); 258 int get_kvm_amd_param_integer(const char *param); 259 260 unsigned int kvm_check_cap(long cap); 261 262 static inline bool kvm_has_cap(long cap) 263 { 264 return kvm_check_cap(cap); 265 } 266 267 #define __KVM_SYSCALL_ERROR(_name, _ret) \ 268 "%s failed, rc: %i errno: %i (%s)", (_name), (_ret), errno, strerror(errno) 269 270 /* 271 * Use the "inner", double-underscore macro when reporting errors from within 272 * other macros so that the name of ioctl() and not its literal numeric value 273 * is printed on error. The "outer" macro is strongly preferred when reporting 274 * errors "directly", i.e. without an additional layer of macros, as it reduces 275 * the probability of passing in the wrong string. 276 */ 277 #define __KVM_IOCTL_ERROR(_name, _ret) __KVM_SYSCALL_ERROR(_name, _ret) 278 #define KVM_IOCTL_ERROR(_ioctl, _ret) __KVM_IOCTL_ERROR(#_ioctl, _ret) 279 280 #define kvm_do_ioctl(fd, cmd, arg) \ 281 ({ \ 282 kvm_static_assert(!_IOC_SIZE(cmd) || sizeof(*arg) == _IOC_SIZE(cmd)); \ 283 ioctl(fd, cmd, arg); \ 284 }) 285 286 #define __kvm_ioctl(kvm_fd, cmd, arg) \ 287 kvm_do_ioctl(kvm_fd, cmd, arg) 288 289 #define kvm_ioctl(kvm_fd, cmd, arg) \ 290 ({ \ 291 int ret = __kvm_ioctl(kvm_fd, cmd, arg); \ 292 \ 293 TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(#cmd, ret)); \ 294 }) 295 296 static __always_inline void static_assert_is_vm(struct kvm_vm *vm) { } 297 298 #define __vm_ioctl(vm, cmd, arg) \ 299 ({ \ 300 static_assert_is_vm(vm); \ 301 kvm_do_ioctl((vm)->fd, cmd, arg); \ 302 }) 303 304 /* 305 * Assert that a VM or vCPU ioctl() succeeded, with extra magic to detect if 306 * the ioctl() failed because KVM killed/bugged the VM. To detect a dead VM, 307 * probe KVM_CAP_USER_MEMORY, which (a) has been supported by KVM since before 308 * selftests existed and (b) should never outright fail, i.e. is supposed to 309 * return 0 or 1. If KVM kills a VM, KVM returns -EIO for all ioctl()s for the 310 * VM and its vCPUs, including KVM_CHECK_EXTENSION. 311 */ 312 #define __TEST_ASSERT_VM_VCPU_IOCTL(cond, name, ret, vm) \ 313 do { \ 314 int __errno = errno; \ 315 \ 316 static_assert_is_vm(vm); \ 317 \ 318 if (cond) \ 319 break; \ 320 \ 321 if (errno == EIO && \ 322 __vm_ioctl(vm, KVM_CHECK_EXTENSION, (void *)KVM_CAP_USER_MEMORY) < 0) { \ 323 TEST_ASSERT(errno == EIO, "KVM killed the VM, should return -EIO"); \ 324 TEST_FAIL("KVM killed/bugged the VM, check the kernel log for clues"); \ 325 } \ 326 errno = __errno; \ 327 TEST_ASSERT(cond, __KVM_IOCTL_ERROR(name, ret)); \ 328 } while (0) 329 330 #define TEST_ASSERT_VM_VCPU_IOCTL(cond, cmd, ret, vm) \ 331 __TEST_ASSERT_VM_VCPU_IOCTL(cond, #cmd, ret, vm) 332 333 #define vm_ioctl(vm, cmd, arg) \ 334 ({ \ 335 int ret = __vm_ioctl(vm, cmd, arg); \ 336 \ 337 __TEST_ASSERT_VM_VCPU_IOCTL(!ret, #cmd, ret, vm); \ 338 }) 339 340 static __always_inline void static_assert_is_vcpu(struct kvm_vcpu *vcpu) { } 341 342 #define __vcpu_ioctl(vcpu, cmd, arg) \ 343 ({ \ 344 static_assert_is_vcpu(vcpu); \ 345 kvm_do_ioctl((vcpu)->fd, cmd, arg); \ 346 }) 347 348 #define vcpu_ioctl(vcpu, cmd, arg) \ 349 ({ \ 350 int ret = __vcpu_ioctl(vcpu, cmd, arg); \ 351 \ 352 __TEST_ASSERT_VM_VCPU_IOCTL(!ret, #cmd, ret, (vcpu)->vm); \ 353 }) 354 355 /* 356 * Looks up and returns the value corresponding to the capability 357 * (KVM_CAP_*) given by cap. 358 */ 359 static inline int vm_check_cap(struct kvm_vm *vm, long cap) 360 { 361 int ret = __vm_ioctl(vm, KVM_CHECK_EXTENSION, (void *)cap); 362 363 TEST_ASSERT_VM_VCPU_IOCTL(ret >= 0, KVM_CHECK_EXTENSION, ret, vm); 364 return ret; 365 } 366 367 static inline int __vm_enable_cap(struct kvm_vm *vm, uint32_t cap, uint64_t arg0) 368 { 369 struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } }; 370 371 return __vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap); 372 } 373 static inline void vm_enable_cap(struct kvm_vm *vm, uint32_t cap, uint64_t arg0) 374 { 375 struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } }; 376 377 vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap); 378 } 379 380 static inline void vm_set_memory_attributes(struct kvm_vm *vm, uint64_t gpa, 381 uint64_t size, uint64_t attributes) 382 { 383 struct kvm_memory_attributes attr = { 384 .attributes = attributes, 385 .address = gpa, 386 .size = size, 387 .flags = 0, 388 }; 389 390 /* 391 * KVM_SET_MEMORY_ATTRIBUTES overwrites _all_ attributes. These flows 392 * need significant enhancements to support multiple attributes. 393 */ 394 TEST_ASSERT(!attributes || attributes == KVM_MEMORY_ATTRIBUTE_PRIVATE, 395 "Update me to support multiple attributes!"); 396 397 vm_ioctl(vm, KVM_SET_MEMORY_ATTRIBUTES, &attr); 398 } 399 400 401 static inline void vm_mem_set_private(struct kvm_vm *vm, uint64_t gpa, 402 uint64_t size) 403 { 404 vm_set_memory_attributes(vm, gpa, size, KVM_MEMORY_ATTRIBUTE_PRIVATE); 405 } 406 407 static inline void vm_mem_set_shared(struct kvm_vm *vm, uint64_t gpa, 408 uint64_t size) 409 { 410 vm_set_memory_attributes(vm, gpa, size, 0); 411 } 412 413 void vm_guest_mem_fallocate(struct kvm_vm *vm, uint64_t gpa, uint64_t size, 414 bool punch_hole); 415 416 static inline void vm_guest_mem_punch_hole(struct kvm_vm *vm, uint64_t gpa, 417 uint64_t size) 418 { 419 vm_guest_mem_fallocate(vm, gpa, size, true); 420 } 421 422 static inline void vm_guest_mem_allocate(struct kvm_vm *vm, uint64_t gpa, 423 uint64_t size) 424 { 425 vm_guest_mem_fallocate(vm, gpa, size, false); 426 } 427 428 void vm_enable_dirty_ring(struct kvm_vm *vm, uint32_t ring_size); 429 const char *vm_guest_mode_string(uint32_t i); 430 431 void kvm_vm_free(struct kvm_vm *vmp); 432 void kvm_vm_restart(struct kvm_vm *vmp); 433 void kvm_vm_release(struct kvm_vm *vmp); 434 int kvm_memcmp_hva_gva(void *hva, struct kvm_vm *vm, const vm_vaddr_t gva, 435 size_t len); 436 void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename); 437 int kvm_memfd_alloc(size_t size, bool hugepages); 438 439 void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent); 440 441 static inline void kvm_vm_get_dirty_log(struct kvm_vm *vm, int slot, void *log) 442 { 443 struct kvm_dirty_log args = { .dirty_bitmap = log, .slot = slot }; 444 445 vm_ioctl(vm, KVM_GET_DIRTY_LOG, &args); 446 } 447 448 static inline void kvm_vm_clear_dirty_log(struct kvm_vm *vm, int slot, void *log, 449 uint64_t first_page, uint32_t num_pages) 450 { 451 struct kvm_clear_dirty_log args = { 452 .dirty_bitmap = log, 453 .slot = slot, 454 .first_page = first_page, 455 .num_pages = num_pages 456 }; 457 458 vm_ioctl(vm, KVM_CLEAR_DIRTY_LOG, &args); 459 } 460 461 static inline uint32_t kvm_vm_reset_dirty_ring(struct kvm_vm *vm) 462 { 463 return __vm_ioctl(vm, KVM_RESET_DIRTY_RINGS, NULL); 464 } 465 466 static inline int vm_get_stats_fd(struct kvm_vm *vm) 467 { 468 int fd = __vm_ioctl(vm, KVM_GET_STATS_FD, NULL); 469 470 TEST_ASSERT_VM_VCPU_IOCTL(fd >= 0, KVM_GET_STATS_FD, fd, vm); 471 return fd; 472 } 473 474 static inline void read_stats_header(int stats_fd, struct kvm_stats_header *header) 475 { 476 ssize_t ret; 477 478 ret = pread(stats_fd, header, sizeof(*header), 0); 479 TEST_ASSERT(ret == sizeof(*header), 480 "Failed to read '%lu' header bytes, ret = '%ld'", 481 sizeof(*header), ret); 482 } 483 484 struct kvm_stats_desc *read_stats_descriptors(int stats_fd, 485 struct kvm_stats_header *header); 486 487 static inline ssize_t get_stats_descriptor_size(struct kvm_stats_header *header) 488 { 489 /* 490 * The base size of the descriptor is defined by KVM's ABI, but the 491 * size of the name field is variable, as far as KVM's ABI is 492 * concerned. For a given instance of KVM, the name field is the same 493 * size for all stats and is provided in the overall stats header. 494 */ 495 return sizeof(struct kvm_stats_desc) + header->name_size; 496 } 497 498 static inline struct kvm_stats_desc *get_stats_descriptor(struct kvm_stats_desc *stats, 499 int index, 500 struct kvm_stats_header *header) 501 { 502 /* 503 * Note, size_desc includes the size of the name field, which is 504 * variable. i.e. this is NOT equivalent to &stats_desc[i]. 505 */ 506 return (void *)stats + index * get_stats_descriptor_size(header); 507 } 508 509 void read_stat_data(int stats_fd, struct kvm_stats_header *header, 510 struct kvm_stats_desc *desc, uint64_t *data, 511 size_t max_elements); 512 513 void __vm_get_stat(struct kvm_vm *vm, const char *stat_name, uint64_t *data, 514 size_t max_elements); 515 516 static inline uint64_t vm_get_stat(struct kvm_vm *vm, const char *stat_name) 517 { 518 uint64_t data; 519 520 __vm_get_stat(vm, stat_name, &data, 1); 521 return data; 522 } 523 524 void vm_create_irqchip(struct kvm_vm *vm); 525 526 static inline int __vm_create_guest_memfd(struct kvm_vm *vm, uint64_t size, 527 uint64_t flags) 528 { 529 struct kvm_create_guest_memfd guest_memfd = { 530 .size = size, 531 .flags = flags, 532 }; 533 534 return __vm_ioctl(vm, KVM_CREATE_GUEST_MEMFD, &guest_memfd); 535 } 536 537 static inline int vm_create_guest_memfd(struct kvm_vm *vm, uint64_t size, 538 uint64_t flags) 539 { 540 int fd = __vm_create_guest_memfd(vm, size, flags); 541 542 TEST_ASSERT(fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_GUEST_MEMFD, fd)); 543 return fd; 544 } 545 546 void vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags, 547 uint64_t gpa, uint64_t size, void *hva); 548 int __vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags, 549 uint64_t gpa, uint64_t size, void *hva); 550 void vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags, 551 uint64_t gpa, uint64_t size, void *hva, 552 uint32_t guest_memfd, uint64_t guest_memfd_offset); 553 int __vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags, 554 uint64_t gpa, uint64_t size, void *hva, 555 uint32_t guest_memfd, uint64_t guest_memfd_offset); 556 557 void vm_userspace_mem_region_add(struct kvm_vm *vm, 558 enum vm_mem_backing_src_type src_type, 559 uint64_t guest_paddr, uint32_t slot, uint64_t npages, 560 uint32_t flags); 561 void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, 562 uint64_t guest_paddr, uint32_t slot, uint64_t npages, 563 uint32_t flags, int guest_memfd_fd, uint64_t guest_memfd_offset); 564 565 #ifndef vm_arch_has_protected_memory 566 static inline bool vm_arch_has_protected_memory(struct kvm_vm *vm) 567 { 568 return false; 569 } 570 #endif 571 572 void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags); 573 void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa); 574 void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot); 575 struct kvm_vcpu *__vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id); 576 void vm_populate_vaddr_bitmap(struct kvm_vm *vm); 577 vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min); 578 vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min); 579 vm_vaddr_t __vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min, 580 enum kvm_mem_region_type type); 581 vm_vaddr_t vm_vaddr_alloc_shared(struct kvm_vm *vm, size_t sz, 582 vm_vaddr_t vaddr_min, 583 enum kvm_mem_region_type type); 584 vm_vaddr_t vm_vaddr_alloc_pages(struct kvm_vm *vm, int nr_pages); 585 vm_vaddr_t __vm_vaddr_alloc_page(struct kvm_vm *vm, 586 enum kvm_mem_region_type type); 587 vm_vaddr_t vm_vaddr_alloc_page(struct kvm_vm *vm); 588 589 void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, 590 unsigned int npages); 591 void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa); 592 void *addr_gva2hva(struct kvm_vm *vm, vm_vaddr_t gva); 593 vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva); 594 void *addr_gpa2alias(struct kvm_vm *vm, vm_paddr_t gpa); 595 596 #ifndef vcpu_arch_put_guest 597 #define vcpu_arch_put_guest(mem, val) do { (mem) = (val); } while (0) 598 #endif 599 600 static inline vm_paddr_t vm_untag_gpa(struct kvm_vm *vm, vm_paddr_t gpa) 601 { 602 return gpa & ~vm->gpa_tag_mask; 603 } 604 605 void vcpu_run(struct kvm_vcpu *vcpu); 606 int _vcpu_run(struct kvm_vcpu *vcpu); 607 608 static inline int __vcpu_run(struct kvm_vcpu *vcpu) 609 { 610 return __vcpu_ioctl(vcpu, KVM_RUN, NULL); 611 } 612 613 void vcpu_run_complete_io(struct kvm_vcpu *vcpu); 614 struct kvm_reg_list *vcpu_get_reg_list(struct kvm_vcpu *vcpu); 615 616 static inline void vcpu_enable_cap(struct kvm_vcpu *vcpu, uint32_t cap, 617 uint64_t arg0) 618 { 619 struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } }; 620 621 vcpu_ioctl(vcpu, KVM_ENABLE_CAP, &enable_cap); 622 } 623 624 static inline void vcpu_guest_debug_set(struct kvm_vcpu *vcpu, 625 struct kvm_guest_debug *debug) 626 { 627 vcpu_ioctl(vcpu, KVM_SET_GUEST_DEBUG, debug); 628 } 629 630 static inline void vcpu_mp_state_get(struct kvm_vcpu *vcpu, 631 struct kvm_mp_state *mp_state) 632 { 633 vcpu_ioctl(vcpu, KVM_GET_MP_STATE, mp_state); 634 } 635 static inline void vcpu_mp_state_set(struct kvm_vcpu *vcpu, 636 struct kvm_mp_state *mp_state) 637 { 638 vcpu_ioctl(vcpu, KVM_SET_MP_STATE, mp_state); 639 } 640 641 static inline void vcpu_regs_get(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 642 { 643 vcpu_ioctl(vcpu, KVM_GET_REGS, regs); 644 } 645 646 static inline void vcpu_regs_set(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 647 { 648 vcpu_ioctl(vcpu, KVM_SET_REGS, regs); 649 } 650 static inline void vcpu_sregs_get(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 651 { 652 vcpu_ioctl(vcpu, KVM_GET_SREGS, sregs); 653 654 } 655 static inline void vcpu_sregs_set(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 656 { 657 vcpu_ioctl(vcpu, KVM_SET_SREGS, sregs); 658 } 659 static inline int _vcpu_sregs_set(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 660 { 661 return __vcpu_ioctl(vcpu, KVM_SET_SREGS, sregs); 662 } 663 static inline void vcpu_fpu_get(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 664 { 665 vcpu_ioctl(vcpu, KVM_GET_FPU, fpu); 666 } 667 static inline void vcpu_fpu_set(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 668 { 669 vcpu_ioctl(vcpu, KVM_SET_FPU, fpu); 670 } 671 672 static inline int __vcpu_get_reg(struct kvm_vcpu *vcpu, uint64_t id, void *addr) 673 { 674 struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)addr }; 675 676 return __vcpu_ioctl(vcpu, KVM_GET_ONE_REG, ®); 677 } 678 static inline int __vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t id, uint64_t val) 679 { 680 struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)&val }; 681 682 return __vcpu_ioctl(vcpu, KVM_SET_ONE_REG, ®); 683 } 684 static inline void vcpu_get_reg(struct kvm_vcpu *vcpu, uint64_t id, void *addr) 685 { 686 struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)addr }; 687 688 vcpu_ioctl(vcpu, KVM_GET_ONE_REG, ®); 689 } 690 static inline void vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t id, uint64_t val) 691 { 692 struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)&val }; 693 694 vcpu_ioctl(vcpu, KVM_SET_ONE_REG, ®); 695 } 696 697 #ifdef __KVM_HAVE_VCPU_EVENTS 698 static inline void vcpu_events_get(struct kvm_vcpu *vcpu, 699 struct kvm_vcpu_events *events) 700 { 701 vcpu_ioctl(vcpu, KVM_GET_VCPU_EVENTS, events); 702 } 703 static inline void vcpu_events_set(struct kvm_vcpu *vcpu, 704 struct kvm_vcpu_events *events) 705 { 706 vcpu_ioctl(vcpu, KVM_SET_VCPU_EVENTS, events); 707 } 708 #endif 709 #ifdef __x86_64__ 710 static inline void vcpu_nested_state_get(struct kvm_vcpu *vcpu, 711 struct kvm_nested_state *state) 712 { 713 vcpu_ioctl(vcpu, KVM_GET_NESTED_STATE, state); 714 } 715 static inline int __vcpu_nested_state_set(struct kvm_vcpu *vcpu, 716 struct kvm_nested_state *state) 717 { 718 return __vcpu_ioctl(vcpu, KVM_SET_NESTED_STATE, state); 719 } 720 721 static inline void vcpu_nested_state_set(struct kvm_vcpu *vcpu, 722 struct kvm_nested_state *state) 723 { 724 vcpu_ioctl(vcpu, KVM_SET_NESTED_STATE, state); 725 } 726 #endif 727 static inline int vcpu_get_stats_fd(struct kvm_vcpu *vcpu) 728 { 729 int fd = __vcpu_ioctl(vcpu, KVM_GET_STATS_FD, NULL); 730 731 TEST_ASSERT_VM_VCPU_IOCTL(fd >= 0, KVM_CHECK_EXTENSION, fd, vcpu->vm); 732 return fd; 733 } 734 735 int __kvm_has_device_attr(int dev_fd, uint32_t group, uint64_t attr); 736 737 static inline void kvm_has_device_attr(int dev_fd, uint32_t group, uint64_t attr) 738 { 739 int ret = __kvm_has_device_attr(dev_fd, group, attr); 740 741 TEST_ASSERT(!ret, "KVM_HAS_DEVICE_ATTR failed, rc: %i errno: %i", ret, errno); 742 } 743 744 int __kvm_device_attr_get(int dev_fd, uint32_t group, uint64_t attr, void *val); 745 746 static inline void kvm_device_attr_get(int dev_fd, uint32_t group, 747 uint64_t attr, void *val) 748 { 749 int ret = __kvm_device_attr_get(dev_fd, group, attr, val); 750 751 TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_GET_DEVICE_ATTR, ret)); 752 } 753 754 int __kvm_device_attr_set(int dev_fd, uint32_t group, uint64_t attr, void *val); 755 756 static inline void kvm_device_attr_set(int dev_fd, uint32_t group, 757 uint64_t attr, void *val) 758 { 759 int ret = __kvm_device_attr_set(dev_fd, group, attr, val); 760 761 TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_SET_DEVICE_ATTR, ret)); 762 } 763 764 static inline int __vcpu_has_device_attr(struct kvm_vcpu *vcpu, uint32_t group, 765 uint64_t attr) 766 { 767 return __kvm_has_device_attr(vcpu->fd, group, attr); 768 } 769 770 static inline void vcpu_has_device_attr(struct kvm_vcpu *vcpu, uint32_t group, 771 uint64_t attr) 772 { 773 kvm_has_device_attr(vcpu->fd, group, attr); 774 } 775 776 static inline int __vcpu_device_attr_get(struct kvm_vcpu *vcpu, uint32_t group, 777 uint64_t attr, void *val) 778 { 779 return __kvm_device_attr_get(vcpu->fd, group, attr, val); 780 } 781 782 static inline void vcpu_device_attr_get(struct kvm_vcpu *vcpu, uint32_t group, 783 uint64_t attr, void *val) 784 { 785 kvm_device_attr_get(vcpu->fd, group, attr, val); 786 } 787 788 static inline int __vcpu_device_attr_set(struct kvm_vcpu *vcpu, uint32_t group, 789 uint64_t attr, void *val) 790 { 791 return __kvm_device_attr_set(vcpu->fd, group, attr, val); 792 } 793 794 static inline void vcpu_device_attr_set(struct kvm_vcpu *vcpu, uint32_t group, 795 uint64_t attr, void *val) 796 { 797 kvm_device_attr_set(vcpu->fd, group, attr, val); 798 } 799 800 int __kvm_test_create_device(struct kvm_vm *vm, uint64_t type); 801 int __kvm_create_device(struct kvm_vm *vm, uint64_t type); 802 803 static inline int kvm_create_device(struct kvm_vm *vm, uint64_t type) 804 { 805 int fd = __kvm_create_device(vm, type); 806 807 TEST_ASSERT(fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_DEVICE, fd)); 808 return fd; 809 } 810 811 void *vcpu_map_dirty_ring(struct kvm_vcpu *vcpu); 812 813 /* 814 * VM VCPU Args Set 815 * 816 * Input Args: 817 * vm - Virtual Machine 818 * num - number of arguments 819 * ... - arguments, each of type uint64_t 820 * 821 * Output Args: None 822 * 823 * Return: None 824 * 825 * Sets the first @num input parameters for the function at @vcpu's entry point, 826 * per the C calling convention of the architecture, to the values given as 827 * variable args. Each of the variable args is expected to be of type uint64_t. 828 * The maximum @num can be is specific to the architecture. 829 */ 830 void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...); 831 832 void kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level); 833 int _kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level); 834 835 #define KVM_MAX_IRQ_ROUTES 4096 836 837 struct kvm_irq_routing *kvm_gsi_routing_create(void); 838 void kvm_gsi_routing_irqchip_add(struct kvm_irq_routing *routing, 839 uint32_t gsi, uint32_t pin); 840 int _kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing); 841 void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing); 842 843 const char *exit_reason_str(unsigned int exit_reason); 844 845 vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min, 846 uint32_t memslot); 847 vm_paddr_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, 848 vm_paddr_t paddr_min, uint32_t memslot, 849 bool protected); 850 vm_paddr_t vm_alloc_page_table(struct kvm_vm *vm); 851 852 static inline vm_paddr_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, 853 vm_paddr_t paddr_min, uint32_t memslot) 854 { 855 /* 856 * By default, allocate memory as protected for VMs that support 857 * protected memory, as the majority of memory for such VMs is 858 * protected, i.e. using shared memory is effectively opt-in. 859 */ 860 return __vm_phy_pages_alloc(vm, num, paddr_min, memslot, 861 vm_arch_has_protected_memory(vm)); 862 } 863 864 /* 865 * ____vm_create() does KVM_CREATE_VM and little else. __vm_create() also 866 * loads the test binary into guest memory and creates an IRQ chip (x86 only). 867 * __vm_create() does NOT create vCPUs, @nr_runnable_vcpus is used purely to 868 * calculate the amount of memory needed for per-vCPU data, e.g. stacks. 869 */ 870 struct kvm_vm *____vm_create(struct vm_shape shape); 871 struct kvm_vm *__vm_create(struct vm_shape shape, uint32_t nr_runnable_vcpus, 872 uint64_t nr_extra_pages); 873 874 static inline struct kvm_vm *vm_create_barebones(void) 875 { 876 return ____vm_create(VM_SHAPE_DEFAULT); 877 } 878 879 static inline struct kvm_vm *vm_create_barebones_type(unsigned long type) 880 { 881 const struct vm_shape shape = { 882 .mode = VM_MODE_DEFAULT, 883 .type = type, 884 }; 885 886 return ____vm_create(shape); 887 } 888 889 static inline struct kvm_vm *vm_create(uint32_t nr_runnable_vcpus) 890 { 891 return __vm_create(VM_SHAPE_DEFAULT, nr_runnable_vcpus, 0); 892 } 893 894 struct kvm_vm *__vm_create_with_vcpus(struct vm_shape shape, uint32_t nr_vcpus, 895 uint64_t extra_mem_pages, 896 void *guest_code, struct kvm_vcpu *vcpus[]); 897 898 static inline struct kvm_vm *vm_create_with_vcpus(uint32_t nr_vcpus, 899 void *guest_code, 900 struct kvm_vcpu *vcpus[]) 901 { 902 return __vm_create_with_vcpus(VM_SHAPE_DEFAULT, nr_vcpus, 0, 903 guest_code, vcpus); 904 } 905 906 907 struct kvm_vm *__vm_create_shape_with_one_vcpu(struct vm_shape shape, 908 struct kvm_vcpu **vcpu, 909 uint64_t extra_mem_pages, 910 void *guest_code); 911 912 /* 913 * Create a VM with a single vCPU with reasonable defaults and @extra_mem_pages 914 * additional pages of guest memory. Returns the VM and vCPU (via out param). 915 */ 916 static inline struct kvm_vm *__vm_create_with_one_vcpu(struct kvm_vcpu **vcpu, 917 uint64_t extra_mem_pages, 918 void *guest_code) 919 { 920 return __vm_create_shape_with_one_vcpu(VM_SHAPE_DEFAULT, vcpu, 921 extra_mem_pages, guest_code); 922 } 923 924 static inline struct kvm_vm *vm_create_with_one_vcpu(struct kvm_vcpu **vcpu, 925 void *guest_code) 926 { 927 return __vm_create_with_one_vcpu(vcpu, 0, guest_code); 928 } 929 930 static inline struct kvm_vm *vm_create_shape_with_one_vcpu(struct vm_shape shape, 931 struct kvm_vcpu **vcpu, 932 void *guest_code) 933 { 934 return __vm_create_shape_with_one_vcpu(shape, vcpu, 0, guest_code); 935 } 936 937 struct kvm_vcpu *vm_recreate_with_one_vcpu(struct kvm_vm *vm); 938 939 void kvm_pin_this_task_to_pcpu(uint32_t pcpu); 940 void kvm_print_vcpu_pinning_help(void); 941 void kvm_parse_vcpu_pinning(const char *pcpus_string, uint32_t vcpu_to_pcpu[], 942 int nr_vcpus); 943 944 unsigned long vm_compute_max_gfn(struct kvm_vm *vm); 945 unsigned int vm_calc_num_guest_pages(enum vm_guest_mode mode, size_t size); 946 unsigned int vm_num_host_pages(enum vm_guest_mode mode, unsigned int num_guest_pages); 947 unsigned int vm_num_guest_pages(enum vm_guest_mode mode, unsigned int num_host_pages); 948 static inline unsigned int 949 vm_adjust_num_guest_pages(enum vm_guest_mode mode, unsigned int num_guest_pages) 950 { 951 unsigned int n; 952 n = vm_num_guest_pages(mode, vm_num_host_pages(mode, num_guest_pages)); 953 #ifdef __s390x__ 954 /* s390 requires 1M aligned guest sizes */ 955 n = (n + 255) & ~255; 956 #endif 957 return n; 958 } 959 960 #define sync_global_to_guest(vm, g) ({ \ 961 typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g)); \ 962 memcpy(_p, &(g), sizeof(g)); \ 963 }) 964 965 #define sync_global_from_guest(vm, g) ({ \ 966 typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g)); \ 967 memcpy(&(g), _p, sizeof(g)); \ 968 }) 969 970 /* 971 * Write a global value, but only in the VM's (guest's) domain. Primarily used 972 * for "globals" that hold per-VM values (VMs always duplicate code and global 973 * data into their own region of physical memory), but can be used anytime it's 974 * undesirable to change the host's copy of the global. 975 */ 976 #define write_guest_global(vm, g, val) ({ \ 977 typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g)); \ 978 typeof(g) _val = val; \ 979 \ 980 memcpy(_p, &(_val), sizeof(g)); \ 981 }) 982 983 void assert_on_unhandled_exception(struct kvm_vcpu *vcpu); 984 985 void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, 986 uint8_t indent); 987 988 static inline void vcpu_dump(FILE *stream, struct kvm_vcpu *vcpu, 989 uint8_t indent) 990 { 991 vcpu_arch_dump(stream, vcpu, indent); 992 } 993 994 /* 995 * Adds a vCPU with reasonable defaults (e.g. a stack) 996 * 997 * Input Args: 998 * vm - Virtual Machine 999 * vcpu_id - The id of the VCPU to add to the VM. 1000 */ 1001 struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id); 1002 void vcpu_arch_set_entry_point(struct kvm_vcpu *vcpu, void *guest_code); 1003 1004 static inline struct kvm_vcpu *vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id, 1005 void *guest_code) 1006 { 1007 struct kvm_vcpu *vcpu = vm_arch_vcpu_add(vm, vcpu_id); 1008 1009 vcpu_arch_set_entry_point(vcpu, guest_code); 1010 1011 return vcpu; 1012 } 1013 1014 /* Re-create a vCPU after restarting a VM, e.g. for state save/restore tests. */ 1015 struct kvm_vcpu *vm_arch_vcpu_recreate(struct kvm_vm *vm, uint32_t vcpu_id); 1016 1017 static inline struct kvm_vcpu *vm_vcpu_recreate(struct kvm_vm *vm, 1018 uint32_t vcpu_id) 1019 { 1020 return vm_arch_vcpu_recreate(vm, vcpu_id); 1021 } 1022 1023 void vcpu_arch_free(struct kvm_vcpu *vcpu); 1024 1025 void virt_arch_pgd_alloc(struct kvm_vm *vm); 1026 1027 static inline void virt_pgd_alloc(struct kvm_vm *vm) 1028 { 1029 virt_arch_pgd_alloc(vm); 1030 } 1031 1032 /* 1033 * VM Virtual Page Map 1034 * 1035 * Input Args: 1036 * vm - Virtual Machine 1037 * vaddr - VM Virtual Address 1038 * paddr - VM Physical Address 1039 * memslot - Memory region slot for new virtual translation tables 1040 * 1041 * Output Args: None 1042 * 1043 * Return: None 1044 * 1045 * Within @vm, creates a virtual translation for the page starting 1046 * at @vaddr to the page starting at @paddr. 1047 */ 1048 void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr); 1049 1050 static inline void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr) 1051 { 1052 virt_arch_pg_map(vm, vaddr, paddr); 1053 } 1054 1055 1056 /* 1057 * Address Guest Virtual to Guest Physical 1058 * 1059 * Input Args: 1060 * vm - Virtual Machine 1061 * gva - VM virtual address 1062 * 1063 * Output Args: None 1064 * 1065 * Return: 1066 * Equivalent VM physical address 1067 * 1068 * Returns the VM physical address of the translated VM virtual 1069 * address given by @gva. 1070 */ 1071 vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva); 1072 1073 static inline vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva) 1074 { 1075 return addr_arch_gva2gpa(vm, gva); 1076 } 1077 1078 /* 1079 * Virtual Translation Tables Dump 1080 * 1081 * Input Args: 1082 * stream - Output FILE stream 1083 * vm - Virtual Machine 1084 * indent - Left margin indent amount 1085 * 1086 * Output Args: None 1087 * 1088 * Return: None 1089 * 1090 * Dumps to the FILE stream given by @stream, the contents of all the 1091 * virtual translation tables for the VM given by @vm. 1092 */ 1093 void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent); 1094 1095 static inline void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent) 1096 { 1097 virt_arch_dump(stream, vm, indent); 1098 } 1099 1100 1101 static inline int __vm_disable_nx_huge_pages(struct kvm_vm *vm) 1102 { 1103 return __vm_enable_cap(vm, KVM_CAP_VM_DISABLE_NX_HUGE_PAGES, 0); 1104 } 1105 1106 /* 1107 * Arch hook that is invoked via a constructor, i.e. before exeucting main(), 1108 * to allow for arch-specific setup that is common to all tests, e.g. computing 1109 * the default guest "mode". 1110 */ 1111 void kvm_selftest_arch_init(void); 1112 1113 void kvm_arch_vm_post_create(struct kvm_vm *vm); 1114 1115 bool vm_is_gpa_protected(struct kvm_vm *vm, vm_paddr_t paddr); 1116 1117 uint32_t guest_get_vcpuid(void); 1118 1119 #endif /* SELFTEST_KVM_UTIL_H */ 1120