1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2018, Google LLC. 4 */ 5 #ifndef SELFTEST_KVM_UTIL_H 6 #define SELFTEST_KVM_UTIL_H 7 8 #include "test_util.h" 9 10 #include <linux/compiler.h> 11 #include "linux/hashtable.h" 12 #include "linux/list.h" 13 #include <linux/kernel.h> 14 #include <linux/kvm.h> 15 #include "linux/rbtree.h" 16 #include <linux/types.h> 17 18 #include <asm/atomic.h> 19 #include <asm/kvm.h> 20 21 #include <sys/ioctl.h> 22 23 #include "kvm_util_arch.h" 24 #include "kvm_util_types.h" 25 #include "sparsebit.h" 26 27 #define KVM_DEV_PATH "/dev/kvm" 28 #define KVM_MAX_VCPUS 512 29 30 #define NSEC_PER_SEC 1000000000L 31 32 struct userspace_mem_region { 33 struct kvm_userspace_memory_region2 region; 34 struct sparsebit *unused_phy_pages; 35 struct sparsebit *protected_phy_pages; 36 int fd; 37 off_t offset; 38 enum vm_mem_backing_src_type backing_src_type; 39 void *host_mem; 40 void *host_alias; 41 void *mmap_start; 42 void *mmap_alias; 43 size_t mmap_size; 44 struct rb_node gpa_node; 45 struct rb_node hva_node; 46 struct hlist_node slot_node; 47 }; 48 49 struct kvm_binary_stats { 50 int fd; 51 struct kvm_stats_header header; 52 struct kvm_stats_desc *desc; 53 }; 54 55 struct kvm_vcpu { 56 struct list_head list; 57 uint32_t id; 58 int fd; 59 struct kvm_vm *vm; 60 struct kvm_run *run; 61 #ifdef __x86_64__ 62 struct kvm_cpuid2 *cpuid; 63 #endif 64 struct kvm_dirty_gfn *dirty_gfns; 65 uint32_t fetch_index; 66 uint32_t dirty_gfns_count; 67 }; 68 69 struct userspace_mem_regions { 70 struct rb_root gpa_tree; 71 struct rb_root hva_tree; 72 DECLARE_HASHTABLE(slot_hash, 9); 73 }; 74 75 enum kvm_mem_region_type { 76 MEM_REGION_CODE, 77 MEM_REGION_DATA, 78 MEM_REGION_PT, 79 MEM_REGION_TEST_DATA, 80 NR_MEM_REGIONS, 81 }; 82 83 struct kvm_vm { 84 int mode; 85 unsigned long type; 86 int kvm_fd; 87 int fd; 88 unsigned int pgtable_levels; 89 unsigned int page_size; 90 unsigned int page_shift; 91 unsigned int pa_bits; 92 unsigned int va_bits; 93 uint64_t max_gfn; 94 struct list_head vcpus; 95 struct userspace_mem_regions regions; 96 struct sparsebit *vpages_valid; 97 struct sparsebit *vpages_mapped; 98 bool has_irqchip; 99 bool pgd_created; 100 vm_paddr_t ucall_mmio_addr; 101 vm_paddr_t pgd; 102 vm_vaddr_t handlers; 103 uint32_t dirty_ring_size; 104 uint64_t gpa_tag_mask; 105 106 struct kvm_vm_arch arch; 107 108 struct kvm_binary_stats stats; 109 110 /* 111 * KVM region slots. These are the default memslots used by page 112 * allocators, e.g., lib/elf uses the memslots[MEM_REGION_CODE] 113 * memslot. 114 */ 115 uint32_t memslots[NR_MEM_REGIONS]; 116 }; 117 118 struct vcpu_reg_sublist { 119 const char *name; 120 long capability; 121 int feature; 122 int feature_type; 123 bool finalize; 124 __u64 *regs; 125 __u64 regs_n; 126 __u64 *rejects_set; 127 __u64 rejects_set_n; 128 __u64 *skips_set; 129 __u64 skips_set_n; 130 }; 131 132 struct vcpu_reg_list { 133 char *name; 134 struct vcpu_reg_sublist sublists[]; 135 }; 136 137 #define for_each_sublist(c, s) \ 138 for ((s) = &(c)->sublists[0]; (s)->regs; ++(s)) 139 140 #define kvm_for_each_vcpu(vm, i, vcpu) \ 141 for ((i) = 0; (i) <= (vm)->last_vcpu_id; (i)++) \ 142 if (!((vcpu) = vm->vcpus[i])) \ 143 continue; \ 144 else 145 146 struct userspace_mem_region * 147 memslot2region(struct kvm_vm *vm, uint32_t memslot); 148 149 static inline struct userspace_mem_region *vm_get_mem_region(struct kvm_vm *vm, 150 enum kvm_mem_region_type type) 151 { 152 assert(type < NR_MEM_REGIONS); 153 return memslot2region(vm, vm->memslots[type]); 154 } 155 156 /* Minimum allocated guest virtual and physical addresses */ 157 #define KVM_UTIL_MIN_VADDR 0x2000 158 #define KVM_GUEST_PAGE_TABLE_MIN_PADDR 0x180000 159 160 #define DEFAULT_GUEST_STACK_VADDR_MIN 0xab6000 161 #define DEFAULT_STACK_PGS 5 162 163 enum vm_guest_mode { 164 VM_MODE_P52V48_4K, 165 VM_MODE_P52V48_16K, 166 VM_MODE_P52V48_64K, 167 VM_MODE_P48V48_4K, 168 VM_MODE_P48V48_16K, 169 VM_MODE_P48V48_64K, 170 VM_MODE_P40V48_4K, 171 VM_MODE_P40V48_16K, 172 VM_MODE_P40V48_64K, 173 VM_MODE_PXXV48_4K, /* For 48bits VA but ANY bits PA */ 174 VM_MODE_P47V64_4K, 175 VM_MODE_P44V64_4K, 176 VM_MODE_P36V48_4K, 177 VM_MODE_P36V48_16K, 178 VM_MODE_P36V48_64K, 179 VM_MODE_P36V47_16K, 180 NUM_VM_MODES, 181 }; 182 183 struct vm_shape { 184 uint32_t type; 185 uint8_t mode; 186 uint8_t pad0; 187 uint16_t pad1; 188 }; 189 190 kvm_static_assert(sizeof(struct vm_shape) == sizeof(uint64_t)); 191 192 #define VM_TYPE_DEFAULT 0 193 194 #define VM_SHAPE(__mode) \ 195 ({ \ 196 struct vm_shape shape = { \ 197 .mode = (__mode), \ 198 .type = VM_TYPE_DEFAULT \ 199 }; \ 200 \ 201 shape; \ 202 }) 203 204 #if defined(__aarch64__) 205 206 extern enum vm_guest_mode vm_mode_default; 207 208 #define VM_MODE_DEFAULT vm_mode_default 209 #define MIN_PAGE_SHIFT 12U 210 #define ptes_per_page(page_size) ((page_size) / 8) 211 212 #elif defined(__x86_64__) 213 214 #define VM_MODE_DEFAULT VM_MODE_PXXV48_4K 215 #define MIN_PAGE_SHIFT 12U 216 #define ptes_per_page(page_size) ((page_size) / 8) 217 218 #elif defined(__s390x__) 219 220 #define VM_MODE_DEFAULT VM_MODE_P44V64_4K 221 #define MIN_PAGE_SHIFT 12U 222 #define ptes_per_page(page_size) ((page_size) / 16) 223 224 #elif defined(__riscv) 225 226 #if __riscv_xlen == 32 227 #error "RISC-V 32-bit kvm selftests not supported" 228 #endif 229 230 #define VM_MODE_DEFAULT VM_MODE_P40V48_4K 231 #define MIN_PAGE_SHIFT 12U 232 #define ptes_per_page(page_size) ((page_size) / 8) 233 234 #endif 235 236 #define VM_SHAPE_DEFAULT VM_SHAPE(VM_MODE_DEFAULT) 237 238 #define MIN_PAGE_SIZE (1U << MIN_PAGE_SHIFT) 239 #define PTES_PER_MIN_PAGE ptes_per_page(MIN_PAGE_SIZE) 240 241 struct vm_guest_mode_params { 242 unsigned int pa_bits; 243 unsigned int va_bits; 244 unsigned int page_size; 245 unsigned int page_shift; 246 }; 247 extern const struct vm_guest_mode_params vm_guest_mode_params[]; 248 249 int open_path_or_exit(const char *path, int flags); 250 int open_kvm_dev_path_or_exit(void); 251 252 bool get_kvm_param_bool(const char *param); 253 bool get_kvm_intel_param_bool(const char *param); 254 bool get_kvm_amd_param_bool(const char *param); 255 256 int get_kvm_param_integer(const char *param); 257 int get_kvm_intel_param_integer(const char *param); 258 int get_kvm_amd_param_integer(const char *param); 259 260 unsigned int kvm_check_cap(long cap); 261 262 static inline bool kvm_has_cap(long cap) 263 { 264 return kvm_check_cap(cap); 265 } 266 267 #define __KVM_SYSCALL_ERROR(_name, _ret) \ 268 "%s failed, rc: %i errno: %i (%s)", (_name), (_ret), errno, strerror(errno) 269 270 /* 271 * Use the "inner", double-underscore macro when reporting errors from within 272 * other macros so that the name of ioctl() and not its literal numeric value 273 * is printed on error. The "outer" macro is strongly preferred when reporting 274 * errors "directly", i.e. without an additional layer of macros, as it reduces 275 * the probability of passing in the wrong string. 276 */ 277 #define __KVM_IOCTL_ERROR(_name, _ret) __KVM_SYSCALL_ERROR(_name, _ret) 278 #define KVM_IOCTL_ERROR(_ioctl, _ret) __KVM_IOCTL_ERROR(#_ioctl, _ret) 279 280 #define kvm_do_ioctl(fd, cmd, arg) \ 281 ({ \ 282 kvm_static_assert(!_IOC_SIZE(cmd) || sizeof(*arg) == _IOC_SIZE(cmd)); \ 283 ioctl(fd, cmd, arg); \ 284 }) 285 286 #define __kvm_ioctl(kvm_fd, cmd, arg) \ 287 kvm_do_ioctl(kvm_fd, cmd, arg) 288 289 #define kvm_ioctl(kvm_fd, cmd, arg) \ 290 ({ \ 291 int ret = __kvm_ioctl(kvm_fd, cmd, arg); \ 292 \ 293 TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(#cmd, ret)); \ 294 }) 295 296 static __always_inline void static_assert_is_vm(struct kvm_vm *vm) { } 297 298 #define __vm_ioctl(vm, cmd, arg) \ 299 ({ \ 300 static_assert_is_vm(vm); \ 301 kvm_do_ioctl((vm)->fd, cmd, arg); \ 302 }) 303 304 /* 305 * Assert that a VM or vCPU ioctl() succeeded, with extra magic to detect if 306 * the ioctl() failed because KVM killed/bugged the VM. To detect a dead VM, 307 * probe KVM_CAP_USER_MEMORY, which (a) has been supported by KVM since before 308 * selftests existed and (b) should never outright fail, i.e. is supposed to 309 * return 0 or 1. If KVM kills a VM, KVM returns -EIO for all ioctl()s for the 310 * VM and its vCPUs, including KVM_CHECK_EXTENSION. 311 */ 312 #define __TEST_ASSERT_VM_VCPU_IOCTL(cond, name, ret, vm) \ 313 do { \ 314 int __errno = errno; \ 315 \ 316 static_assert_is_vm(vm); \ 317 \ 318 if (cond) \ 319 break; \ 320 \ 321 if (errno == EIO && \ 322 __vm_ioctl(vm, KVM_CHECK_EXTENSION, (void *)KVM_CAP_USER_MEMORY) < 0) { \ 323 TEST_ASSERT(errno == EIO, "KVM killed the VM, should return -EIO"); \ 324 TEST_FAIL("KVM killed/bugged the VM, check the kernel log for clues"); \ 325 } \ 326 errno = __errno; \ 327 TEST_ASSERT(cond, __KVM_IOCTL_ERROR(name, ret)); \ 328 } while (0) 329 330 #define TEST_ASSERT_VM_VCPU_IOCTL(cond, cmd, ret, vm) \ 331 __TEST_ASSERT_VM_VCPU_IOCTL(cond, #cmd, ret, vm) 332 333 #define vm_ioctl(vm, cmd, arg) \ 334 ({ \ 335 int ret = __vm_ioctl(vm, cmd, arg); \ 336 \ 337 __TEST_ASSERT_VM_VCPU_IOCTL(!ret, #cmd, ret, vm); \ 338 }) 339 340 static __always_inline void static_assert_is_vcpu(struct kvm_vcpu *vcpu) { } 341 342 #define __vcpu_ioctl(vcpu, cmd, arg) \ 343 ({ \ 344 static_assert_is_vcpu(vcpu); \ 345 kvm_do_ioctl((vcpu)->fd, cmd, arg); \ 346 }) 347 348 #define vcpu_ioctl(vcpu, cmd, arg) \ 349 ({ \ 350 int ret = __vcpu_ioctl(vcpu, cmd, arg); \ 351 \ 352 __TEST_ASSERT_VM_VCPU_IOCTL(!ret, #cmd, ret, (vcpu)->vm); \ 353 }) 354 355 /* 356 * Looks up and returns the value corresponding to the capability 357 * (KVM_CAP_*) given by cap. 358 */ 359 static inline int vm_check_cap(struct kvm_vm *vm, long cap) 360 { 361 int ret = __vm_ioctl(vm, KVM_CHECK_EXTENSION, (void *)cap); 362 363 TEST_ASSERT_VM_VCPU_IOCTL(ret >= 0, KVM_CHECK_EXTENSION, ret, vm); 364 return ret; 365 } 366 367 static inline int __vm_enable_cap(struct kvm_vm *vm, uint32_t cap, uint64_t arg0) 368 { 369 struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } }; 370 371 return __vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap); 372 } 373 static inline void vm_enable_cap(struct kvm_vm *vm, uint32_t cap, uint64_t arg0) 374 { 375 struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } }; 376 377 vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap); 378 } 379 380 static inline void vm_set_memory_attributes(struct kvm_vm *vm, uint64_t gpa, 381 uint64_t size, uint64_t attributes) 382 { 383 struct kvm_memory_attributes attr = { 384 .attributes = attributes, 385 .address = gpa, 386 .size = size, 387 .flags = 0, 388 }; 389 390 /* 391 * KVM_SET_MEMORY_ATTRIBUTES overwrites _all_ attributes. These flows 392 * need significant enhancements to support multiple attributes. 393 */ 394 TEST_ASSERT(!attributes || attributes == KVM_MEMORY_ATTRIBUTE_PRIVATE, 395 "Update me to support multiple attributes!"); 396 397 vm_ioctl(vm, KVM_SET_MEMORY_ATTRIBUTES, &attr); 398 } 399 400 401 static inline void vm_mem_set_private(struct kvm_vm *vm, uint64_t gpa, 402 uint64_t size) 403 { 404 vm_set_memory_attributes(vm, gpa, size, KVM_MEMORY_ATTRIBUTE_PRIVATE); 405 } 406 407 static inline void vm_mem_set_shared(struct kvm_vm *vm, uint64_t gpa, 408 uint64_t size) 409 { 410 vm_set_memory_attributes(vm, gpa, size, 0); 411 } 412 413 void vm_guest_mem_fallocate(struct kvm_vm *vm, uint64_t gpa, uint64_t size, 414 bool punch_hole); 415 416 static inline void vm_guest_mem_punch_hole(struct kvm_vm *vm, uint64_t gpa, 417 uint64_t size) 418 { 419 vm_guest_mem_fallocate(vm, gpa, size, true); 420 } 421 422 static inline void vm_guest_mem_allocate(struct kvm_vm *vm, uint64_t gpa, 423 uint64_t size) 424 { 425 vm_guest_mem_fallocate(vm, gpa, size, false); 426 } 427 428 void vm_enable_dirty_ring(struct kvm_vm *vm, uint32_t ring_size); 429 const char *vm_guest_mode_string(uint32_t i); 430 431 void kvm_vm_free(struct kvm_vm *vmp); 432 void kvm_vm_restart(struct kvm_vm *vmp); 433 void kvm_vm_release(struct kvm_vm *vmp); 434 void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename); 435 int kvm_memfd_alloc(size_t size, bool hugepages); 436 437 void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent); 438 439 static inline void kvm_vm_get_dirty_log(struct kvm_vm *vm, int slot, void *log) 440 { 441 struct kvm_dirty_log args = { .dirty_bitmap = log, .slot = slot }; 442 443 vm_ioctl(vm, KVM_GET_DIRTY_LOG, &args); 444 } 445 446 static inline void kvm_vm_clear_dirty_log(struct kvm_vm *vm, int slot, void *log, 447 uint64_t first_page, uint32_t num_pages) 448 { 449 struct kvm_clear_dirty_log args = { 450 .dirty_bitmap = log, 451 .slot = slot, 452 .first_page = first_page, 453 .num_pages = num_pages 454 }; 455 456 vm_ioctl(vm, KVM_CLEAR_DIRTY_LOG, &args); 457 } 458 459 static inline uint32_t kvm_vm_reset_dirty_ring(struct kvm_vm *vm) 460 { 461 return __vm_ioctl(vm, KVM_RESET_DIRTY_RINGS, NULL); 462 } 463 464 static inline void kvm_vm_register_coalesced_io(struct kvm_vm *vm, 465 uint64_t address, 466 uint64_t size, bool pio) 467 { 468 struct kvm_coalesced_mmio_zone zone = { 469 .addr = address, 470 .size = size, 471 .pio = pio, 472 }; 473 474 vm_ioctl(vm, KVM_REGISTER_COALESCED_MMIO, &zone); 475 } 476 477 static inline void kvm_vm_unregister_coalesced_io(struct kvm_vm *vm, 478 uint64_t address, 479 uint64_t size, bool pio) 480 { 481 struct kvm_coalesced_mmio_zone zone = { 482 .addr = address, 483 .size = size, 484 .pio = pio, 485 }; 486 487 vm_ioctl(vm, KVM_UNREGISTER_COALESCED_MMIO, &zone); 488 } 489 490 static inline int vm_get_stats_fd(struct kvm_vm *vm) 491 { 492 int fd = __vm_ioctl(vm, KVM_GET_STATS_FD, NULL); 493 494 TEST_ASSERT_VM_VCPU_IOCTL(fd >= 0, KVM_GET_STATS_FD, fd, vm); 495 return fd; 496 } 497 498 static inline void read_stats_header(int stats_fd, struct kvm_stats_header *header) 499 { 500 ssize_t ret; 501 502 ret = pread(stats_fd, header, sizeof(*header), 0); 503 TEST_ASSERT(ret == sizeof(*header), 504 "Failed to read '%lu' header bytes, ret = '%ld'", 505 sizeof(*header), ret); 506 } 507 508 struct kvm_stats_desc *read_stats_descriptors(int stats_fd, 509 struct kvm_stats_header *header); 510 511 static inline ssize_t get_stats_descriptor_size(struct kvm_stats_header *header) 512 { 513 /* 514 * The base size of the descriptor is defined by KVM's ABI, but the 515 * size of the name field is variable, as far as KVM's ABI is 516 * concerned. For a given instance of KVM, the name field is the same 517 * size for all stats and is provided in the overall stats header. 518 */ 519 return sizeof(struct kvm_stats_desc) + header->name_size; 520 } 521 522 static inline struct kvm_stats_desc *get_stats_descriptor(struct kvm_stats_desc *stats, 523 int index, 524 struct kvm_stats_header *header) 525 { 526 /* 527 * Note, size_desc includes the size of the name field, which is 528 * variable. i.e. this is NOT equivalent to &stats_desc[i]. 529 */ 530 return (void *)stats + index * get_stats_descriptor_size(header); 531 } 532 533 void read_stat_data(int stats_fd, struct kvm_stats_header *header, 534 struct kvm_stats_desc *desc, uint64_t *data, 535 size_t max_elements); 536 537 void __vm_get_stat(struct kvm_vm *vm, const char *stat_name, uint64_t *data, 538 size_t max_elements); 539 540 #define vm_get_stat(vm, stat) \ 541 ({ \ 542 uint64_t data; \ 543 \ 544 __vm_get_stat(vm, #stat, &data, 1); \ 545 data; \ 546 }) 547 548 void vm_create_irqchip(struct kvm_vm *vm); 549 550 static inline int __vm_create_guest_memfd(struct kvm_vm *vm, uint64_t size, 551 uint64_t flags) 552 { 553 struct kvm_create_guest_memfd guest_memfd = { 554 .size = size, 555 .flags = flags, 556 }; 557 558 return __vm_ioctl(vm, KVM_CREATE_GUEST_MEMFD, &guest_memfd); 559 } 560 561 static inline int vm_create_guest_memfd(struct kvm_vm *vm, uint64_t size, 562 uint64_t flags) 563 { 564 int fd = __vm_create_guest_memfd(vm, size, flags); 565 566 TEST_ASSERT(fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_GUEST_MEMFD, fd)); 567 return fd; 568 } 569 570 void vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags, 571 uint64_t gpa, uint64_t size, void *hva); 572 int __vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags, 573 uint64_t gpa, uint64_t size, void *hva); 574 void vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags, 575 uint64_t gpa, uint64_t size, void *hva, 576 uint32_t guest_memfd, uint64_t guest_memfd_offset); 577 int __vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags, 578 uint64_t gpa, uint64_t size, void *hva, 579 uint32_t guest_memfd, uint64_t guest_memfd_offset); 580 581 void vm_userspace_mem_region_add(struct kvm_vm *vm, 582 enum vm_mem_backing_src_type src_type, 583 uint64_t guest_paddr, uint32_t slot, uint64_t npages, 584 uint32_t flags); 585 void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, 586 uint64_t guest_paddr, uint32_t slot, uint64_t npages, 587 uint32_t flags, int guest_memfd_fd, uint64_t guest_memfd_offset); 588 589 #ifndef vm_arch_has_protected_memory 590 static inline bool vm_arch_has_protected_memory(struct kvm_vm *vm) 591 { 592 return false; 593 } 594 #endif 595 596 void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags); 597 void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa); 598 void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot); 599 struct kvm_vcpu *__vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id); 600 void vm_populate_vaddr_bitmap(struct kvm_vm *vm); 601 vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min); 602 vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min); 603 vm_vaddr_t __vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min, 604 enum kvm_mem_region_type type); 605 vm_vaddr_t vm_vaddr_alloc_shared(struct kvm_vm *vm, size_t sz, 606 vm_vaddr_t vaddr_min, 607 enum kvm_mem_region_type type); 608 vm_vaddr_t vm_vaddr_alloc_pages(struct kvm_vm *vm, int nr_pages); 609 vm_vaddr_t __vm_vaddr_alloc_page(struct kvm_vm *vm, 610 enum kvm_mem_region_type type); 611 vm_vaddr_t vm_vaddr_alloc_page(struct kvm_vm *vm); 612 613 void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, 614 unsigned int npages); 615 void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa); 616 void *addr_gva2hva(struct kvm_vm *vm, vm_vaddr_t gva); 617 vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva); 618 void *addr_gpa2alias(struct kvm_vm *vm, vm_paddr_t gpa); 619 620 #ifndef vcpu_arch_put_guest 621 #define vcpu_arch_put_guest(mem, val) do { (mem) = (val); } while (0) 622 #endif 623 624 static inline vm_paddr_t vm_untag_gpa(struct kvm_vm *vm, vm_paddr_t gpa) 625 { 626 return gpa & ~vm->gpa_tag_mask; 627 } 628 629 void vcpu_run(struct kvm_vcpu *vcpu); 630 int _vcpu_run(struct kvm_vcpu *vcpu); 631 632 static inline int __vcpu_run(struct kvm_vcpu *vcpu) 633 { 634 return __vcpu_ioctl(vcpu, KVM_RUN, NULL); 635 } 636 637 void vcpu_run_complete_io(struct kvm_vcpu *vcpu); 638 struct kvm_reg_list *vcpu_get_reg_list(struct kvm_vcpu *vcpu); 639 640 static inline void vcpu_enable_cap(struct kvm_vcpu *vcpu, uint32_t cap, 641 uint64_t arg0) 642 { 643 struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } }; 644 645 vcpu_ioctl(vcpu, KVM_ENABLE_CAP, &enable_cap); 646 } 647 648 static inline void vcpu_guest_debug_set(struct kvm_vcpu *vcpu, 649 struct kvm_guest_debug *debug) 650 { 651 vcpu_ioctl(vcpu, KVM_SET_GUEST_DEBUG, debug); 652 } 653 654 static inline void vcpu_mp_state_get(struct kvm_vcpu *vcpu, 655 struct kvm_mp_state *mp_state) 656 { 657 vcpu_ioctl(vcpu, KVM_GET_MP_STATE, mp_state); 658 } 659 static inline void vcpu_mp_state_set(struct kvm_vcpu *vcpu, 660 struct kvm_mp_state *mp_state) 661 { 662 vcpu_ioctl(vcpu, KVM_SET_MP_STATE, mp_state); 663 } 664 665 static inline void vcpu_regs_get(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 666 { 667 vcpu_ioctl(vcpu, KVM_GET_REGS, regs); 668 } 669 670 static inline void vcpu_regs_set(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 671 { 672 vcpu_ioctl(vcpu, KVM_SET_REGS, regs); 673 } 674 static inline void vcpu_sregs_get(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 675 { 676 vcpu_ioctl(vcpu, KVM_GET_SREGS, sregs); 677 678 } 679 static inline void vcpu_sregs_set(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 680 { 681 vcpu_ioctl(vcpu, KVM_SET_SREGS, sregs); 682 } 683 static inline int _vcpu_sregs_set(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 684 { 685 return __vcpu_ioctl(vcpu, KVM_SET_SREGS, sregs); 686 } 687 static inline void vcpu_fpu_get(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 688 { 689 vcpu_ioctl(vcpu, KVM_GET_FPU, fpu); 690 } 691 static inline void vcpu_fpu_set(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 692 { 693 vcpu_ioctl(vcpu, KVM_SET_FPU, fpu); 694 } 695 696 static inline int __vcpu_get_reg(struct kvm_vcpu *vcpu, uint64_t id, void *addr) 697 { 698 struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)addr }; 699 700 return __vcpu_ioctl(vcpu, KVM_GET_ONE_REG, ®); 701 } 702 static inline int __vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t id, uint64_t val) 703 { 704 struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)&val }; 705 706 return __vcpu_ioctl(vcpu, KVM_SET_ONE_REG, ®); 707 } 708 static inline uint64_t vcpu_get_reg(struct kvm_vcpu *vcpu, uint64_t id) 709 { 710 uint64_t val; 711 struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)&val }; 712 713 TEST_ASSERT(KVM_REG_SIZE(id) <= sizeof(val), "Reg %lx too big", id); 714 715 vcpu_ioctl(vcpu, KVM_GET_ONE_REG, ®); 716 return val; 717 } 718 static inline void vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t id, uint64_t val) 719 { 720 struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)&val }; 721 722 TEST_ASSERT(KVM_REG_SIZE(id) <= sizeof(val), "Reg %lx too big", id); 723 724 vcpu_ioctl(vcpu, KVM_SET_ONE_REG, ®); 725 } 726 727 #ifdef __KVM_HAVE_VCPU_EVENTS 728 static inline void vcpu_events_get(struct kvm_vcpu *vcpu, 729 struct kvm_vcpu_events *events) 730 { 731 vcpu_ioctl(vcpu, KVM_GET_VCPU_EVENTS, events); 732 } 733 static inline void vcpu_events_set(struct kvm_vcpu *vcpu, 734 struct kvm_vcpu_events *events) 735 { 736 vcpu_ioctl(vcpu, KVM_SET_VCPU_EVENTS, events); 737 } 738 #endif 739 #ifdef __x86_64__ 740 static inline void vcpu_nested_state_get(struct kvm_vcpu *vcpu, 741 struct kvm_nested_state *state) 742 { 743 vcpu_ioctl(vcpu, KVM_GET_NESTED_STATE, state); 744 } 745 static inline int __vcpu_nested_state_set(struct kvm_vcpu *vcpu, 746 struct kvm_nested_state *state) 747 { 748 return __vcpu_ioctl(vcpu, KVM_SET_NESTED_STATE, state); 749 } 750 751 static inline void vcpu_nested_state_set(struct kvm_vcpu *vcpu, 752 struct kvm_nested_state *state) 753 { 754 vcpu_ioctl(vcpu, KVM_SET_NESTED_STATE, state); 755 } 756 #endif 757 static inline int vcpu_get_stats_fd(struct kvm_vcpu *vcpu) 758 { 759 int fd = __vcpu_ioctl(vcpu, KVM_GET_STATS_FD, NULL); 760 761 TEST_ASSERT_VM_VCPU_IOCTL(fd >= 0, KVM_CHECK_EXTENSION, fd, vcpu->vm); 762 return fd; 763 } 764 765 int __kvm_has_device_attr(int dev_fd, uint32_t group, uint64_t attr); 766 767 static inline void kvm_has_device_attr(int dev_fd, uint32_t group, uint64_t attr) 768 { 769 int ret = __kvm_has_device_attr(dev_fd, group, attr); 770 771 TEST_ASSERT(!ret, "KVM_HAS_DEVICE_ATTR failed, rc: %i errno: %i", ret, errno); 772 } 773 774 int __kvm_device_attr_get(int dev_fd, uint32_t group, uint64_t attr, void *val); 775 776 static inline void kvm_device_attr_get(int dev_fd, uint32_t group, 777 uint64_t attr, void *val) 778 { 779 int ret = __kvm_device_attr_get(dev_fd, group, attr, val); 780 781 TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_GET_DEVICE_ATTR, ret)); 782 } 783 784 int __kvm_device_attr_set(int dev_fd, uint32_t group, uint64_t attr, void *val); 785 786 static inline void kvm_device_attr_set(int dev_fd, uint32_t group, 787 uint64_t attr, void *val) 788 { 789 int ret = __kvm_device_attr_set(dev_fd, group, attr, val); 790 791 TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_SET_DEVICE_ATTR, ret)); 792 } 793 794 static inline int __vcpu_has_device_attr(struct kvm_vcpu *vcpu, uint32_t group, 795 uint64_t attr) 796 { 797 return __kvm_has_device_attr(vcpu->fd, group, attr); 798 } 799 800 static inline void vcpu_has_device_attr(struct kvm_vcpu *vcpu, uint32_t group, 801 uint64_t attr) 802 { 803 kvm_has_device_attr(vcpu->fd, group, attr); 804 } 805 806 static inline int __vcpu_device_attr_get(struct kvm_vcpu *vcpu, uint32_t group, 807 uint64_t attr, void *val) 808 { 809 return __kvm_device_attr_get(vcpu->fd, group, attr, val); 810 } 811 812 static inline void vcpu_device_attr_get(struct kvm_vcpu *vcpu, uint32_t group, 813 uint64_t attr, void *val) 814 { 815 kvm_device_attr_get(vcpu->fd, group, attr, val); 816 } 817 818 static inline int __vcpu_device_attr_set(struct kvm_vcpu *vcpu, uint32_t group, 819 uint64_t attr, void *val) 820 { 821 return __kvm_device_attr_set(vcpu->fd, group, attr, val); 822 } 823 824 static inline void vcpu_device_attr_set(struct kvm_vcpu *vcpu, uint32_t group, 825 uint64_t attr, void *val) 826 { 827 kvm_device_attr_set(vcpu->fd, group, attr, val); 828 } 829 830 int __kvm_test_create_device(struct kvm_vm *vm, uint64_t type); 831 int __kvm_create_device(struct kvm_vm *vm, uint64_t type); 832 833 static inline int kvm_create_device(struct kvm_vm *vm, uint64_t type) 834 { 835 int fd = __kvm_create_device(vm, type); 836 837 TEST_ASSERT(fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_DEVICE, fd)); 838 return fd; 839 } 840 841 void *vcpu_map_dirty_ring(struct kvm_vcpu *vcpu); 842 843 /* 844 * VM VCPU Args Set 845 * 846 * Input Args: 847 * vm - Virtual Machine 848 * num - number of arguments 849 * ... - arguments, each of type uint64_t 850 * 851 * Output Args: None 852 * 853 * Return: None 854 * 855 * Sets the first @num input parameters for the function at @vcpu's entry point, 856 * per the C calling convention of the architecture, to the values given as 857 * variable args. Each of the variable args is expected to be of type uint64_t. 858 * The maximum @num can be is specific to the architecture. 859 */ 860 void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...); 861 862 void kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level); 863 int _kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level); 864 865 #define KVM_MAX_IRQ_ROUTES 4096 866 867 struct kvm_irq_routing *kvm_gsi_routing_create(void); 868 void kvm_gsi_routing_irqchip_add(struct kvm_irq_routing *routing, 869 uint32_t gsi, uint32_t pin); 870 int _kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing); 871 void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing); 872 873 const char *exit_reason_str(unsigned int exit_reason); 874 875 vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min, 876 uint32_t memslot); 877 vm_paddr_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, 878 vm_paddr_t paddr_min, uint32_t memslot, 879 bool protected); 880 vm_paddr_t vm_alloc_page_table(struct kvm_vm *vm); 881 882 static inline vm_paddr_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, 883 vm_paddr_t paddr_min, uint32_t memslot) 884 { 885 /* 886 * By default, allocate memory as protected for VMs that support 887 * protected memory, as the majority of memory for such VMs is 888 * protected, i.e. using shared memory is effectively opt-in. 889 */ 890 return __vm_phy_pages_alloc(vm, num, paddr_min, memslot, 891 vm_arch_has_protected_memory(vm)); 892 } 893 894 /* 895 * ____vm_create() does KVM_CREATE_VM and little else. __vm_create() also 896 * loads the test binary into guest memory and creates an IRQ chip (x86 only). 897 * __vm_create() does NOT create vCPUs, @nr_runnable_vcpus is used purely to 898 * calculate the amount of memory needed for per-vCPU data, e.g. stacks. 899 */ 900 struct kvm_vm *____vm_create(struct vm_shape shape); 901 struct kvm_vm *__vm_create(struct vm_shape shape, uint32_t nr_runnable_vcpus, 902 uint64_t nr_extra_pages); 903 904 static inline struct kvm_vm *vm_create_barebones(void) 905 { 906 return ____vm_create(VM_SHAPE_DEFAULT); 907 } 908 909 static inline struct kvm_vm *vm_create_barebones_type(unsigned long type) 910 { 911 const struct vm_shape shape = { 912 .mode = VM_MODE_DEFAULT, 913 .type = type, 914 }; 915 916 return ____vm_create(shape); 917 } 918 919 static inline struct kvm_vm *vm_create(uint32_t nr_runnable_vcpus) 920 { 921 return __vm_create(VM_SHAPE_DEFAULT, nr_runnable_vcpus, 0); 922 } 923 924 struct kvm_vm *__vm_create_with_vcpus(struct vm_shape shape, uint32_t nr_vcpus, 925 uint64_t extra_mem_pages, 926 void *guest_code, struct kvm_vcpu *vcpus[]); 927 928 static inline struct kvm_vm *vm_create_with_vcpus(uint32_t nr_vcpus, 929 void *guest_code, 930 struct kvm_vcpu *vcpus[]) 931 { 932 return __vm_create_with_vcpus(VM_SHAPE_DEFAULT, nr_vcpus, 0, 933 guest_code, vcpus); 934 } 935 936 937 struct kvm_vm *__vm_create_shape_with_one_vcpu(struct vm_shape shape, 938 struct kvm_vcpu **vcpu, 939 uint64_t extra_mem_pages, 940 void *guest_code); 941 942 /* 943 * Create a VM with a single vCPU with reasonable defaults and @extra_mem_pages 944 * additional pages of guest memory. Returns the VM and vCPU (via out param). 945 */ 946 static inline struct kvm_vm *__vm_create_with_one_vcpu(struct kvm_vcpu **vcpu, 947 uint64_t extra_mem_pages, 948 void *guest_code) 949 { 950 return __vm_create_shape_with_one_vcpu(VM_SHAPE_DEFAULT, vcpu, 951 extra_mem_pages, guest_code); 952 } 953 954 static inline struct kvm_vm *vm_create_with_one_vcpu(struct kvm_vcpu **vcpu, 955 void *guest_code) 956 { 957 return __vm_create_with_one_vcpu(vcpu, 0, guest_code); 958 } 959 960 static inline struct kvm_vm *vm_create_shape_with_one_vcpu(struct vm_shape shape, 961 struct kvm_vcpu **vcpu, 962 void *guest_code) 963 { 964 return __vm_create_shape_with_one_vcpu(shape, vcpu, 0, guest_code); 965 } 966 967 struct kvm_vcpu *vm_recreate_with_one_vcpu(struct kvm_vm *vm); 968 969 void kvm_set_files_rlimit(uint32_t nr_vcpus); 970 971 void kvm_pin_this_task_to_pcpu(uint32_t pcpu); 972 void kvm_print_vcpu_pinning_help(void); 973 void kvm_parse_vcpu_pinning(const char *pcpus_string, uint32_t vcpu_to_pcpu[], 974 int nr_vcpus); 975 976 unsigned long vm_compute_max_gfn(struct kvm_vm *vm); 977 unsigned int vm_calc_num_guest_pages(enum vm_guest_mode mode, size_t size); 978 unsigned int vm_num_host_pages(enum vm_guest_mode mode, unsigned int num_guest_pages); 979 unsigned int vm_num_guest_pages(enum vm_guest_mode mode, unsigned int num_host_pages); 980 static inline unsigned int 981 vm_adjust_num_guest_pages(enum vm_guest_mode mode, unsigned int num_guest_pages) 982 { 983 unsigned int n; 984 n = vm_num_guest_pages(mode, vm_num_host_pages(mode, num_guest_pages)); 985 #ifdef __s390x__ 986 /* s390 requires 1M aligned guest sizes */ 987 n = (n + 255) & ~255; 988 #endif 989 return n; 990 } 991 992 #define sync_global_to_guest(vm, g) ({ \ 993 typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g)); \ 994 memcpy(_p, &(g), sizeof(g)); \ 995 }) 996 997 #define sync_global_from_guest(vm, g) ({ \ 998 typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g)); \ 999 memcpy(&(g), _p, sizeof(g)); \ 1000 }) 1001 1002 /* 1003 * Write a global value, but only in the VM's (guest's) domain. Primarily used 1004 * for "globals" that hold per-VM values (VMs always duplicate code and global 1005 * data into their own region of physical memory), but can be used anytime it's 1006 * undesirable to change the host's copy of the global. 1007 */ 1008 #define write_guest_global(vm, g, val) ({ \ 1009 typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g)); \ 1010 typeof(g) _val = val; \ 1011 \ 1012 memcpy(_p, &(_val), sizeof(g)); \ 1013 }) 1014 1015 void assert_on_unhandled_exception(struct kvm_vcpu *vcpu); 1016 1017 void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, 1018 uint8_t indent); 1019 1020 static inline void vcpu_dump(FILE *stream, struct kvm_vcpu *vcpu, 1021 uint8_t indent) 1022 { 1023 vcpu_arch_dump(stream, vcpu, indent); 1024 } 1025 1026 /* 1027 * Adds a vCPU with reasonable defaults (e.g. a stack) 1028 * 1029 * Input Args: 1030 * vm - Virtual Machine 1031 * vcpu_id - The id of the VCPU to add to the VM. 1032 */ 1033 struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id); 1034 void vcpu_arch_set_entry_point(struct kvm_vcpu *vcpu, void *guest_code); 1035 1036 static inline struct kvm_vcpu *vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id, 1037 void *guest_code) 1038 { 1039 struct kvm_vcpu *vcpu = vm_arch_vcpu_add(vm, vcpu_id); 1040 1041 vcpu_arch_set_entry_point(vcpu, guest_code); 1042 1043 return vcpu; 1044 } 1045 1046 /* Re-create a vCPU after restarting a VM, e.g. for state save/restore tests. */ 1047 struct kvm_vcpu *vm_arch_vcpu_recreate(struct kvm_vm *vm, uint32_t vcpu_id); 1048 1049 static inline struct kvm_vcpu *vm_vcpu_recreate(struct kvm_vm *vm, 1050 uint32_t vcpu_id) 1051 { 1052 return vm_arch_vcpu_recreate(vm, vcpu_id); 1053 } 1054 1055 void vcpu_arch_free(struct kvm_vcpu *vcpu); 1056 1057 void virt_arch_pgd_alloc(struct kvm_vm *vm); 1058 1059 static inline void virt_pgd_alloc(struct kvm_vm *vm) 1060 { 1061 virt_arch_pgd_alloc(vm); 1062 } 1063 1064 /* 1065 * VM Virtual Page Map 1066 * 1067 * Input Args: 1068 * vm - Virtual Machine 1069 * vaddr - VM Virtual Address 1070 * paddr - VM Physical Address 1071 * memslot - Memory region slot for new virtual translation tables 1072 * 1073 * Output Args: None 1074 * 1075 * Return: None 1076 * 1077 * Within @vm, creates a virtual translation for the page starting 1078 * at @vaddr to the page starting at @paddr. 1079 */ 1080 void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr); 1081 1082 static inline void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr) 1083 { 1084 virt_arch_pg_map(vm, vaddr, paddr); 1085 } 1086 1087 1088 /* 1089 * Address Guest Virtual to Guest Physical 1090 * 1091 * Input Args: 1092 * vm - Virtual Machine 1093 * gva - VM virtual address 1094 * 1095 * Output Args: None 1096 * 1097 * Return: 1098 * Equivalent VM physical address 1099 * 1100 * Returns the VM physical address of the translated VM virtual 1101 * address given by @gva. 1102 */ 1103 vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva); 1104 1105 static inline vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva) 1106 { 1107 return addr_arch_gva2gpa(vm, gva); 1108 } 1109 1110 /* 1111 * Virtual Translation Tables Dump 1112 * 1113 * Input Args: 1114 * stream - Output FILE stream 1115 * vm - Virtual Machine 1116 * indent - Left margin indent amount 1117 * 1118 * Output Args: None 1119 * 1120 * Return: None 1121 * 1122 * Dumps to the FILE stream given by @stream, the contents of all the 1123 * virtual translation tables for the VM given by @vm. 1124 */ 1125 void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent); 1126 1127 static inline void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent) 1128 { 1129 virt_arch_dump(stream, vm, indent); 1130 } 1131 1132 1133 static inline int __vm_disable_nx_huge_pages(struct kvm_vm *vm) 1134 { 1135 return __vm_enable_cap(vm, KVM_CAP_VM_DISABLE_NX_HUGE_PAGES, 0); 1136 } 1137 1138 /* 1139 * Arch hook that is invoked via a constructor, i.e. before exeucting main(), 1140 * to allow for arch-specific setup that is common to all tests, e.g. computing 1141 * the default guest "mode". 1142 */ 1143 void kvm_selftest_arch_init(void); 1144 1145 void kvm_arch_vm_post_create(struct kvm_vm *vm); 1146 1147 bool vm_is_gpa_protected(struct kvm_vm *vm, vm_paddr_t paddr); 1148 1149 uint32_t guest_get_vcpuid(void); 1150 1151 #endif /* SELFTEST_KVM_UTIL_H */ 1152