1 #ifndef __KVM_HOST_H 2 #define __KVM_HOST_H 3 4 /* 5 * This work is licensed under the terms of the GNU GPL, version 2. See 6 * the COPYING file in the top-level directory. 7 */ 8 9 #include <linux/types.h> 10 #include <linux/hardirq.h> 11 #include <linux/list.h> 12 #include <linux/mutex.h> 13 #include <linux/spinlock.h> 14 #include <linux/signal.h> 15 #include <linux/sched.h> 16 #include <linux/bug.h> 17 #include <linux/mm.h> 18 #include <linux/mmu_notifier.h> 19 #include <linux/preempt.h> 20 #include <linux/msi.h> 21 #include <linux/slab.h> 22 #include <linux/rcupdate.h> 23 #include <linux/ratelimit.h> 24 #include <linux/err.h> 25 #include <asm/signal.h> 26 27 #include <linux/kvm.h> 28 #include <linux/kvm_para.h> 29 30 #include <linux/kvm_types.h> 31 32 #include <asm/kvm_host.h> 33 34 #ifndef KVM_MMIO_SIZE 35 #define KVM_MMIO_SIZE 8 36 #endif 37 38 /* 39 * The bit 16 ~ bit 31 of kvm_memory_region::flags are internally used 40 * in kvm, other bits are visible for userspace which are defined in 41 * include/linux/kvm_h. 42 */ 43 #define KVM_MEMSLOT_INVALID (1UL << 16) 44 45 /* Two fragments for cross MMIO pages. */ 46 #define KVM_MAX_MMIO_FRAGMENTS 2 47 48 /* 49 * For the normal pfn, the highest 12 bits should be zero, 50 * so we can mask these bits to indicate the error. 51 */ 52 #define KVM_PFN_ERR_MASK (0xfffULL << 52) 53 54 #define KVM_PFN_ERR_FAULT (KVM_PFN_ERR_MASK) 55 #define KVM_PFN_ERR_HWPOISON (KVM_PFN_ERR_MASK + 1) 56 #define KVM_PFN_ERR_BAD (KVM_PFN_ERR_MASK + 2) 57 #define KVM_PFN_ERR_RO_FAULT (KVM_PFN_ERR_MASK + 3) 58 59 static inline bool is_error_pfn(pfn_t pfn) 60 { 61 return !!(pfn & KVM_PFN_ERR_MASK); 62 } 63 64 static inline bool is_noslot_pfn(pfn_t pfn) 65 { 66 return pfn == KVM_PFN_ERR_BAD; 67 } 68 69 static inline bool is_invalid_pfn(pfn_t pfn) 70 { 71 return !is_noslot_pfn(pfn) && is_error_pfn(pfn); 72 } 73 74 #define KVM_HVA_ERR_BAD (PAGE_OFFSET) 75 #define KVM_HVA_ERR_RO_BAD (PAGE_OFFSET + PAGE_SIZE) 76 77 static inline bool kvm_is_error_hva(unsigned long addr) 78 { 79 return addr >= PAGE_OFFSET; 80 } 81 82 #define KVM_ERR_PTR_BAD_PAGE (ERR_PTR(-ENOENT)) 83 84 static inline bool is_error_page(struct page *page) 85 { 86 return IS_ERR(page); 87 } 88 89 /* 90 * vcpu->requests bit members 91 */ 92 #define KVM_REQ_TLB_FLUSH 0 93 #define KVM_REQ_MIGRATE_TIMER 1 94 #define KVM_REQ_REPORT_TPR_ACCESS 2 95 #define KVM_REQ_MMU_RELOAD 3 96 #define KVM_REQ_TRIPLE_FAULT 4 97 #define KVM_REQ_PENDING_TIMER 5 98 #define KVM_REQ_UNHALT 6 99 #define KVM_REQ_MMU_SYNC 7 100 #define KVM_REQ_CLOCK_UPDATE 8 101 #define KVM_REQ_KICK 9 102 #define KVM_REQ_DEACTIVATE_FPU 10 103 #define KVM_REQ_EVENT 11 104 #define KVM_REQ_APF_HALT 12 105 #define KVM_REQ_STEAL_UPDATE 13 106 #define KVM_REQ_NMI 14 107 #define KVM_REQ_IMMEDIATE_EXIT 15 108 #define KVM_REQ_PMU 16 109 #define KVM_REQ_PMI 17 110 111 #define KVM_USERSPACE_IRQ_SOURCE_ID 0 112 #define KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID 1 113 114 struct kvm; 115 struct kvm_vcpu; 116 extern struct kmem_cache *kvm_vcpu_cache; 117 118 struct kvm_io_range { 119 gpa_t addr; 120 int len; 121 struct kvm_io_device *dev; 122 }; 123 124 #define NR_IOBUS_DEVS 1000 125 126 struct kvm_io_bus { 127 int dev_count; 128 struct kvm_io_range range[]; 129 }; 130 131 enum kvm_bus { 132 KVM_MMIO_BUS, 133 KVM_PIO_BUS, 134 KVM_NR_BUSES 135 }; 136 137 int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 138 int len, const void *val); 139 int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, int len, 140 void *val); 141 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 142 int len, struct kvm_io_device *dev); 143 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, 144 struct kvm_io_device *dev); 145 146 #ifdef CONFIG_KVM_ASYNC_PF 147 struct kvm_async_pf { 148 struct work_struct work; 149 struct list_head link; 150 struct list_head queue; 151 struct kvm_vcpu *vcpu; 152 struct mm_struct *mm; 153 gva_t gva; 154 unsigned long addr; 155 struct kvm_arch_async_pf arch; 156 struct page *page; 157 bool done; 158 }; 159 160 void kvm_clear_async_pf_completion_queue(struct kvm_vcpu *vcpu); 161 void kvm_check_async_pf_completion(struct kvm_vcpu *vcpu); 162 int kvm_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn, 163 struct kvm_arch_async_pf *arch); 164 int kvm_async_pf_wakeup_all(struct kvm_vcpu *vcpu); 165 #endif 166 167 enum { 168 OUTSIDE_GUEST_MODE, 169 IN_GUEST_MODE, 170 EXITING_GUEST_MODE, 171 READING_SHADOW_PAGE_TABLES, 172 }; 173 174 /* 175 * Sometimes a large or cross-page mmio needs to be broken up into separate 176 * exits for userspace servicing. 177 */ 178 struct kvm_mmio_fragment { 179 gpa_t gpa; 180 void *data; 181 unsigned len; 182 }; 183 184 struct kvm_vcpu { 185 struct kvm *kvm; 186 #ifdef CONFIG_PREEMPT_NOTIFIERS 187 struct preempt_notifier preempt_notifier; 188 #endif 189 int cpu; 190 int vcpu_id; 191 int srcu_idx; 192 int mode; 193 unsigned long requests; 194 unsigned long guest_debug; 195 196 struct mutex mutex; 197 struct kvm_run *run; 198 199 int fpu_active; 200 int guest_fpu_loaded, guest_xcr0_loaded; 201 wait_queue_head_t wq; 202 struct pid *pid; 203 int sigset_active; 204 sigset_t sigset; 205 struct kvm_vcpu_stat stat; 206 207 #ifdef CONFIG_HAS_IOMEM 208 int mmio_needed; 209 int mmio_read_completed; 210 int mmio_is_write; 211 int mmio_cur_fragment; 212 int mmio_nr_fragments; 213 struct kvm_mmio_fragment mmio_fragments[KVM_MAX_MMIO_FRAGMENTS]; 214 #endif 215 216 #ifdef CONFIG_KVM_ASYNC_PF 217 struct { 218 u32 queued; 219 struct list_head queue; 220 struct list_head done; 221 spinlock_t lock; 222 } async_pf; 223 #endif 224 225 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT 226 /* 227 * Cpu relax intercept or pause loop exit optimization 228 * in_spin_loop: set when a vcpu does a pause loop exit 229 * or cpu relax intercepted. 230 * dy_eligible: indicates whether vcpu is eligible for directed yield. 231 */ 232 struct { 233 bool in_spin_loop; 234 bool dy_eligible; 235 } spin_loop; 236 #endif 237 struct kvm_vcpu_arch arch; 238 }; 239 240 static inline int kvm_vcpu_exiting_guest_mode(struct kvm_vcpu *vcpu) 241 { 242 return cmpxchg(&vcpu->mode, IN_GUEST_MODE, EXITING_GUEST_MODE); 243 } 244 245 /* 246 * Some of the bitops functions do not support too long bitmaps. 247 * This number must be determined not to exceed such limits. 248 */ 249 #define KVM_MEM_MAX_NR_PAGES ((1UL << 31) - 1) 250 251 struct kvm_memory_slot { 252 gfn_t base_gfn; 253 unsigned long npages; 254 unsigned long flags; 255 unsigned long *dirty_bitmap; 256 struct kvm_arch_memory_slot arch; 257 unsigned long userspace_addr; 258 int user_alloc; 259 int id; 260 }; 261 262 static inline unsigned long kvm_dirty_bitmap_bytes(struct kvm_memory_slot *memslot) 263 { 264 return ALIGN(memslot->npages, BITS_PER_LONG) / 8; 265 } 266 267 struct kvm_kernel_irq_routing_entry { 268 u32 gsi; 269 u32 type; 270 int (*set)(struct kvm_kernel_irq_routing_entry *e, 271 struct kvm *kvm, int irq_source_id, int level); 272 union { 273 struct { 274 unsigned irqchip; 275 unsigned pin; 276 } irqchip; 277 struct msi_msg msi; 278 }; 279 struct hlist_node link; 280 }; 281 282 #ifdef __KVM_HAVE_IOAPIC 283 284 struct kvm_irq_routing_table { 285 int chip[KVM_NR_IRQCHIPS][KVM_IOAPIC_NUM_PINS]; 286 struct kvm_kernel_irq_routing_entry *rt_entries; 287 u32 nr_rt_entries; 288 /* 289 * Array indexed by gsi. Each entry contains list of irq chips 290 * the gsi is connected to. 291 */ 292 struct hlist_head map[0]; 293 }; 294 295 #else 296 297 struct kvm_irq_routing_table {}; 298 299 #endif 300 301 #ifndef KVM_MEM_SLOTS_NUM 302 #define KVM_MEM_SLOTS_NUM (KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS) 303 #endif 304 305 /* 306 * Note: 307 * memslots are not sorted by id anymore, please use id_to_memslot() 308 * to get the memslot by its id. 309 */ 310 struct kvm_memslots { 311 u64 generation; 312 struct kvm_memory_slot memslots[KVM_MEM_SLOTS_NUM]; 313 /* The mapping table from slot id to the index in memslots[]. */ 314 int id_to_index[KVM_MEM_SLOTS_NUM]; 315 }; 316 317 struct kvm { 318 spinlock_t mmu_lock; 319 struct mutex slots_lock; 320 struct mm_struct *mm; /* userspace tied to this vm */ 321 struct kvm_memslots *memslots; 322 struct srcu_struct srcu; 323 #ifdef CONFIG_KVM_APIC_ARCHITECTURE 324 u32 bsp_vcpu_id; 325 #endif 326 struct kvm_vcpu *vcpus[KVM_MAX_VCPUS]; 327 atomic_t online_vcpus; 328 int last_boosted_vcpu; 329 struct list_head vm_list; 330 struct mutex lock; 331 struct kvm_io_bus *buses[KVM_NR_BUSES]; 332 #ifdef CONFIG_HAVE_KVM_EVENTFD 333 struct { 334 spinlock_t lock; 335 struct list_head items; 336 struct list_head resampler_list; 337 struct mutex resampler_lock; 338 } irqfds; 339 struct list_head ioeventfds; 340 #endif 341 struct kvm_vm_stat stat; 342 struct kvm_arch arch; 343 atomic_t users_count; 344 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 345 struct kvm_coalesced_mmio_ring *coalesced_mmio_ring; 346 spinlock_t ring_lock; 347 struct list_head coalesced_zones; 348 #endif 349 350 struct mutex irq_lock; 351 #ifdef CONFIG_HAVE_KVM_IRQCHIP 352 /* 353 * Update side is protected by irq_lock and, 354 * if configured, irqfds.lock. 355 */ 356 struct kvm_irq_routing_table __rcu *irq_routing; 357 struct hlist_head mask_notifier_list; 358 struct hlist_head irq_ack_notifier_list; 359 #endif 360 361 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 362 struct mmu_notifier mmu_notifier; 363 unsigned long mmu_notifier_seq; 364 long mmu_notifier_count; 365 #endif 366 long tlbs_dirty; 367 }; 368 369 #define kvm_err(fmt, ...) \ 370 pr_err("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__) 371 #define kvm_info(fmt, ...) \ 372 pr_info("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__) 373 #define kvm_debug(fmt, ...) \ 374 pr_debug("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__) 375 #define kvm_pr_unimpl(fmt, ...) \ 376 pr_err_ratelimited("kvm [%i]: " fmt, \ 377 task_tgid_nr(current), ## __VA_ARGS__) 378 379 /* The guest did something we don't support. */ 380 #define vcpu_unimpl(vcpu, fmt, ...) \ 381 kvm_pr_unimpl("vcpu%i " fmt, (vcpu)->vcpu_id, ## __VA_ARGS__) 382 383 static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i) 384 { 385 smp_rmb(); 386 return kvm->vcpus[i]; 387 } 388 389 #define kvm_for_each_vcpu(idx, vcpup, kvm) \ 390 for (idx = 0; \ 391 idx < atomic_read(&kvm->online_vcpus) && \ 392 (vcpup = kvm_get_vcpu(kvm, idx)) != NULL; \ 393 idx++) 394 395 #define kvm_for_each_memslot(memslot, slots) \ 396 for (memslot = &slots->memslots[0]; \ 397 memslot < slots->memslots + KVM_MEM_SLOTS_NUM && memslot->npages;\ 398 memslot++) 399 400 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id); 401 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu); 402 403 int __must_check vcpu_load(struct kvm_vcpu *vcpu); 404 void vcpu_put(struct kvm_vcpu *vcpu); 405 406 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, 407 struct module *module); 408 void kvm_exit(void); 409 410 void kvm_get_kvm(struct kvm *kvm); 411 void kvm_put_kvm(struct kvm *kvm); 412 void update_memslots(struct kvm_memslots *slots, struct kvm_memory_slot *new); 413 414 static inline struct kvm_memslots *kvm_memslots(struct kvm *kvm) 415 { 416 return rcu_dereference_check(kvm->memslots, 417 srcu_read_lock_held(&kvm->srcu) 418 || lockdep_is_held(&kvm->slots_lock)); 419 } 420 421 static inline struct kvm_memory_slot * 422 id_to_memslot(struct kvm_memslots *slots, int id) 423 { 424 int index = slots->id_to_index[id]; 425 struct kvm_memory_slot *slot; 426 427 slot = &slots->memslots[index]; 428 429 WARN_ON(slot->id != id); 430 return slot; 431 } 432 433 int kvm_set_memory_region(struct kvm *kvm, 434 struct kvm_userspace_memory_region *mem, 435 int user_alloc); 436 int __kvm_set_memory_region(struct kvm *kvm, 437 struct kvm_userspace_memory_region *mem, 438 int user_alloc); 439 void kvm_arch_free_memslot(struct kvm_memory_slot *free, 440 struct kvm_memory_slot *dont); 441 int kvm_arch_create_memslot(struct kvm_memory_slot *slot, unsigned long npages); 442 int kvm_arch_prepare_memory_region(struct kvm *kvm, 443 struct kvm_memory_slot *memslot, 444 struct kvm_memory_slot old, 445 struct kvm_userspace_memory_region *mem, 446 int user_alloc); 447 void kvm_arch_commit_memory_region(struct kvm *kvm, 448 struct kvm_userspace_memory_region *mem, 449 struct kvm_memory_slot old, 450 int user_alloc); 451 bool kvm_largepages_enabled(void); 452 void kvm_disable_largepages(void); 453 /* flush all memory translations */ 454 void kvm_arch_flush_shadow_all(struct kvm *kvm); 455 /* flush memory translations pointing to 'slot' */ 456 void kvm_arch_flush_shadow_memslot(struct kvm *kvm, 457 struct kvm_memory_slot *slot); 458 459 int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages, 460 int nr_pages); 461 462 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn); 463 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn); 464 unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot, gfn_t gfn); 465 void kvm_release_page_clean(struct page *page); 466 void kvm_release_page_dirty(struct page *page); 467 void kvm_set_page_dirty(struct page *page); 468 void kvm_set_page_accessed(struct page *page); 469 470 pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn); 471 pfn_t gfn_to_pfn_async(struct kvm *kvm, gfn_t gfn, bool *async, 472 bool write_fault, bool *writable); 473 pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn); 474 pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault, 475 bool *writable); 476 pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn); 477 pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn); 478 479 void kvm_release_pfn_dirty(pfn_t pfn); 480 void kvm_release_pfn_clean(pfn_t pfn); 481 void kvm_set_pfn_dirty(pfn_t pfn); 482 void kvm_set_pfn_accessed(pfn_t pfn); 483 void kvm_get_pfn(pfn_t pfn); 484 485 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, 486 int len); 487 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data, 488 unsigned long len); 489 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len); 490 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 491 void *data, unsigned long len); 492 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data, 493 int offset, int len); 494 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, 495 unsigned long len); 496 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 497 void *data, unsigned long len); 498 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 499 gpa_t gpa); 500 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len); 501 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len); 502 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn); 503 int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn); 504 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn); 505 void mark_page_dirty(struct kvm *kvm, gfn_t gfn); 506 void mark_page_dirty_in_slot(struct kvm *kvm, struct kvm_memory_slot *memslot, 507 gfn_t gfn); 508 509 void kvm_vcpu_block(struct kvm_vcpu *vcpu); 510 void kvm_vcpu_kick(struct kvm_vcpu *vcpu); 511 bool kvm_vcpu_yield_to(struct kvm_vcpu *target); 512 void kvm_vcpu_on_spin(struct kvm_vcpu *vcpu); 513 void kvm_resched(struct kvm_vcpu *vcpu); 514 void kvm_load_guest_fpu(struct kvm_vcpu *vcpu); 515 void kvm_put_guest_fpu(struct kvm_vcpu *vcpu); 516 517 void kvm_flush_remote_tlbs(struct kvm *kvm); 518 void kvm_reload_remote_mmus(struct kvm *kvm); 519 520 long kvm_arch_dev_ioctl(struct file *filp, 521 unsigned int ioctl, unsigned long arg); 522 long kvm_arch_vcpu_ioctl(struct file *filp, 523 unsigned int ioctl, unsigned long arg); 524 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf); 525 526 int kvm_dev_ioctl_check_extension(long ext); 527 528 int kvm_get_dirty_log(struct kvm *kvm, 529 struct kvm_dirty_log *log, int *is_dirty); 530 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, 531 struct kvm_dirty_log *log); 532 533 int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, 534 struct 535 kvm_userspace_memory_region *mem, 536 int user_alloc); 537 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level); 538 long kvm_arch_vm_ioctl(struct file *filp, 539 unsigned int ioctl, unsigned long arg); 540 541 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu); 542 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu); 543 544 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 545 struct kvm_translation *tr); 546 547 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs); 548 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs); 549 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, 550 struct kvm_sregs *sregs); 551 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, 552 struct kvm_sregs *sregs); 553 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 554 struct kvm_mp_state *mp_state); 555 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 556 struct kvm_mp_state *mp_state); 557 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 558 struct kvm_guest_debug *dbg); 559 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run); 560 561 int kvm_arch_init(void *opaque); 562 void kvm_arch_exit(void); 563 564 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu); 565 void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu); 566 567 void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu); 568 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu); 569 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu); 570 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id); 571 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu); 572 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu); 573 574 int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu); 575 int kvm_arch_hardware_enable(void *garbage); 576 void kvm_arch_hardware_disable(void *garbage); 577 int kvm_arch_hardware_setup(void); 578 void kvm_arch_hardware_unsetup(void); 579 void kvm_arch_check_processor_compat(void *rtn); 580 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu); 581 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu); 582 583 void kvm_free_physmem(struct kvm *kvm); 584 585 void *kvm_kvzalloc(unsigned long size); 586 void kvm_kvfree(const void *addr); 587 588 #ifndef __KVM_HAVE_ARCH_VM_ALLOC 589 static inline struct kvm *kvm_arch_alloc_vm(void) 590 { 591 return kzalloc(sizeof(struct kvm), GFP_KERNEL); 592 } 593 594 static inline void kvm_arch_free_vm(struct kvm *kvm) 595 { 596 kfree(kvm); 597 } 598 #endif 599 600 static inline wait_queue_head_t *kvm_arch_vcpu_wq(struct kvm_vcpu *vcpu) 601 { 602 #ifdef __KVM_HAVE_ARCH_WQP 603 return vcpu->arch.wqp; 604 #else 605 return &vcpu->wq; 606 #endif 607 } 608 609 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type); 610 void kvm_arch_destroy_vm(struct kvm *kvm); 611 void kvm_free_all_assigned_devices(struct kvm *kvm); 612 void kvm_arch_sync_events(struct kvm *kvm); 613 614 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu); 615 void kvm_vcpu_kick(struct kvm_vcpu *vcpu); 616 617 bool kvm_is_mmio_pfn(pfn_t pfn); 618 619 struct kvm_irq_ack_notifier { 620 struct hlist_node link; 621 unsigned gsi; 622 void (*irq_acked)(struct kvm_irq_ack_notifier *kian); 623 }; 624 625 struct kvm_assigned_dev_kernel { 626 struct kvm_irq_ack_notifier ack_notifier; 627 struct list_head list; 628 int assigned_dev_id; 629 int host_segnr; 630 int host_busnr; 631 int host_devfn; 632 unsigned int entries_nr; 633 int host_irq; 634 bool host_irq_disabled; 635 bool pci_2_3; 636 struct msix_entry *host_msix_entries; 637 int guest_irq; 638 struct msix_entry *guest_msix_entries; 639 unsigned long irq_requested_type; 640 int irq_source_id; 641 int flags; 642 struct pci_dev *dev; 643 struct kvm *kvm; 644 spinlock_t intx_lock; 645 spinlock_t intx_mask_lock; 646 char irq_name[32]; 647 struct pci_saved_state *pci_saved_state; 648 }; 649 650 struct kvm_irq_mask_notifier { 651 void (*func)(struct kvm_irq_mask_notifier *kimn, bool masked); 652 int irq; 653 struct hlist_node link; 654 }; 655 656 void kvm_register_irq_mask_notifier(struct kvm *kvm, int irq, 657 struct kvm_irq_mask_notifier *kimn); 658 void kvm_unregister_irq_mask_notifier(struct kvm *kvm, int irq, 659 struct kvm_irq_mask_notifier *kimn); 660 void kvm_fire_mask_notifiers(struct kvm *kvm, unsigned irqchip, unsigned pin, 661 bool mask); 662 663 #ifdef __KVM_HAVE_IOAPIC 664 void kvm_get_intr_delivery_bitmask(struct kvm_ioapic *ioapic, 665 union kvm_ioapic_redirect_entry *entry, 666 unsigned long *deliver_bitmask); 667 #endif 668 int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level); 669 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *irq_entry, struct kvm *kvm, 670 int irq_source_id, int level); 671 void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin); 672 void kvm_register_irq_ack_notifier(struct kvm *kvm, 673 struct kvm_irq_ack_notifier *kian); 674 void kvm_unregister_irq_ack_notifier(struct kvm *kvm, 675 struct kvm_irq_ack_notifier *kian); 676 int kvm_request_irq_source_id(struct kvm *kvm); 677 void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id); 678 679 /* For vcpu->arch.iommu_flags */ 680 #define KVM_IOMMU_CACHE_COHERENCY 0x1 681 682 #ifdef CONFIG_IOMMU_API 683 int kvm_iommu_map_pages(struct kvm *kvm, struct kvm_memory_slot *slot); 684 void kvm_iommu_unmap_pages(struct kvm *kvm, struct kvm_memory_slot *slot); 685 int kvm_iommu_map_guest(struct kvm *kvm); 686 int kvm_iommu_unmap_guest(struct kvm *kvm); 687 int kvm_assign_device(struct kvm *kvm, 688 struct kvm_assigned_dev_kernel *assigned_dev); 689 int kvm_deassign_device(struct kvm *kvm, 690 struct kvm_assigned_dev_kernel *assigned_dev); 691 #else /* CONFIG_IOMMU_API */ 692 static inline int kvm_iommu_map_pages(struct kvm *kvm, 693 struct kvm_memory_slot *slot) 694 { 695 return 0; 696 } 697 698 static inline void kvm_iommu_unmap_pages(struct kvm *kvm, 699 struct kvm_memory_slot *slot) 700 { 701 } 702 703 static inline int kvm_iommu_map_guest(struct kvm *kvm) 704 { 705 return -ENODEV; 706 } 707 708 static inline int kvm_iommu_unmap_guest(struct kvm *kvm) 709 { 710 return 0; 711 } 712 713 static inline int kvm_assign_device(struct kvm *kvm, 714 struct kvm_assigned_dev_kernel *assigned_dev) 715 { 716 return 0; 717 } 718 719 static inline int kvm_deassign_device(struct kvm *kvm, 720 struct kvm_assigned_dev_kernel *assigned_dev) 721 { 722 return 0; 723 } 724 #endif /* CONFIG_IOMMU_API */ 725 726 static inline void kvm_guest_enter(void) 727 { 728 BUG_ON(preemptible()); 729 /* 730 * This is running in ioctl context so we can avoid 731 * the call to vtime_account() with its unnecessary idle check. 732 */ 733 vtime_account_system_irqsafe(current); 734 current->flags |= PF_VCPU; 735 /* KVM does not hold any references to rcu protected data when it 736 * switches CPU into a guest mode. In fact switching to a guest mode 737 * is very similar to exiting to userspase from rcu point of view. In 738 * addition CPU may stay in a guest mode for quite a long time (up to 739 * one time slice). Lets treat guest mode as quiescent state, just like 740 * we do with user-mode execution. 741 */ 742 rcu_virt_note_context_switch(smp_processor_id()); 743 } 744 745 static inline void kvm_guest_exit(void) 746 { 747 /* 748 * This is running in ioctl context so we can avoid 749 * the call to vtime_account() with its unnecessary idle check. 750 */ 751 vtime_account_system_irqsafe(current); 752 current->flags &= ~PF_VCPU; 753 } 754 755 /* 756 * search_memslots() and __gfn_to_memslot() are here because they are 757 * used in non-modular code in arch/powerpc/kvm/book3s_hv_rm_mmu.c. 758 * gfn_to_memslot() itself isn't here as an inline because that would 759 * bloat other code too much. 760 */ 761 static inline struct kvm_memory_slot * 762 search_memslots(struct kvm_memslots *slots, gfn_t gfn) 763 { 764 struct kvm_memory_slot *memslot; 765 766 kvm_for_each_memslot(memslot, slots) 767 if (gfn >= memslot->base_gfn && 768 gfn < memslot->base_gfn + memslot->npages) 769 return memslot; 770 771 return NULL; 772 } 773 774 static inline struct kvm_memory_slot * 775 __gfn_to_memslot(struct kvm_memslots *slots, gfn_t gfn) 776 { 777 return search_memslots(slots, gfn); 778 } 779 780 static inline unsigned long 781 __gfn_to_hva_memslot(struct kvm_memory_slot *slot, gfn_t gfn) 782 { 783 return slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE; 784 } 785 786 static inline int memslot_id(struct kvm *kvm, gfn_t gfn) 787 { 788 return gfn_to_memslot(kvm, gfn)->id; 789 } 790 791 static inline gfn_t gfn_to_index(gfn_t gfn, gfn_t base_gfn, int level) 792 { 793 /* KVM_HPAGE_GFN_SHIFT(PT_PAGE_TABLE_LEVEL) must be 0. */ 794 return (gfn >> KVM_HPAGE_GFN_SHIFT(level)) - 795 (base_gfn >> KVM_HPAGE_GFN_SHIFT(level)); 796 } 797 798 static inline gfn_t 799 hva_to_gfn_memslot(unsigned long hva, struct kvm_memory_slot *slot) 800 { 801 gfn_t gfn_offset = (hva - slot->userspace_addr) >> PAGE_SHIFT; 802 803 return slot->base_gfn + gfn_offset; 804 } 805 806 static inline gpa_t gfn_to_gpa(gfn_t gfn) 807 { 808 return (gpa_t)gfn << PAGE_SHIFT; 809 } 810 811 static inline gfn_t gpa_to_gfn(gpa_t gpa) 812 { 813 return (gfn_t)(gpa >> PAGE_SHIFT); 814 } 815 816 static inline hpa_t pfn_to_hpa(pfn_t pfn) 817 { 818 return (hpa_t)pfn << PAGE_SHIFT; 819 } 820 821 static inline void kvm_migrate_timers(struct kvm_vcpu *vcpu) 822 { 823 set_bit(KVM_REQ_MIGRATE_TIMER, &vcpu->requests); 824 } 825 826 enum kvm_stat_kind { 827 KVM_STAT_VM, 828 KVM_STAT_VCPU, 829 }; 830 831 struct kvm_stats_debugfs_item { 832 const char *name; 833 int offset; 834 enum kvm_stat_kind kind; 835 struct dentry *dentry; 836 }; 837 extern struct kvm_stats_debugfs_item debugfs_entries[]; 838 extern struct dentry *kvm_debugfs_dir; 839 840 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 841 static inline int mmu_notifier_retry(struct kvm_vcpu *vcpu, unsigned long mmu_seq) 842 { 843 if (unlikely(vcpu->kvm->mmu_notifier_count)) 844 return 1; 845 /* 846 * Ensure the read of mmu_notifier_count happens before the read 847 * of mmu_notifier_seq. This interacts with the smp_wmb() in 848 * mmu_notifier_invalidate_range_end to make sure that the caller 849 * either sees the old (non-zero) value of mmu_notifier_count or 850 * the new (incremented) value of mmu_notifier_seq. 851 * PowerPC Book3s HV KVM calls this under a per-page lock 852 * rather than under kvm->mmu_lock, for scalability, so 853 * can't rely on kvm->mmu_lock to keep things ordered. 854 */ 855 smp_rmb(); 856 if (vcpu->kvm->mmu_notifier_seq != mmu_seq) 857 return 1; 858 return 0; 859 } 860 #endif 861 862 #ifdef KVM_CAP_IRQ_ROUTING 863 864 #define KVM_MAX_IRQ_ROUTES 1024 865 866 int kvm_setup_default_irq_routing(struct kvm *kvm); 867 int kvm_set_irq_routing(struct kvm *kvm, 868 const struct kvm_irq_routing_entry *entries, 869 unsigned nr, 870 unsigned flags); 871 void kvm_free_irq_routing(struct kvm *kvm); 872 873 int kvm_send_userspace_msi(struct kvm *kvm, struct kvm_msi *msi); 874 875 #else 876 877 static inline void kvm_free_irq_routing(struct kvm *kvm) {} 878 879 #endif 880 881 #ifdef CONFIG_HAVE_KVM_EVENTFD 882 883 void kvm_eventfd_init(struct kvm *kvm); 884 int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args); 885 void kvm_irqfd_release(struct kvm *kvm); 886 void kvm_irq_routing_update(struct kvm *, struct kvm_irq_routing_table *); 887 int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args); 888 889 #else 890 891 static inline void kvm_eventfd_init(struct kvm *kvm) {} 892 893 static inline int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args) 894 { 895 return -EINVAL; 896 } 897 898 static inline void kvm_irqfd_release(struct kvm *kvm) {} 899 900 #ifdef CONFIG_HAVE_KVM_IRQCHIP 901 static inline void kvm_irq_routing_update(struct kvm *kvm, 902 struct kvm_irq_routing_table *irq_rt) 903 { 904 rcu_assign_pointer(kvm->irq_routing, irq_rt); 905 } 906 #endif 907 908 static inline int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args) 909 { 910 return -ENOSYS; 911 } 912 913 #endif /* CONFIG_HAVE_KVM_EVENTFD */ 914 915 #ifdef CONFIG_KVM_APIC_ARCHITECTURE 916 static inline bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu) 917 { 918 return vcpu->kvm->bsp_vcpu_id == vcpu->vcpu_id; 919 } 920 921 bool kvm_vcpu_compatible(struct kvm_vcpu *vcpu); 922 923 #else 924 925 static inline bool kvm_vcpu_compatible(struct kvm_vcpu *vcpu) { return true; } 926 927 #endif 928 929 #ifdef __KVM_HAVE_DEVICE_ASSIGNMENT 930 931 long kvm_vm_ioctl_assigned_device(struct kvm *kvm, unsigned ioctl, 932 unsigned long arg); 933 934 #else 935 936 static inline long kvm_vm_ioctl_assigned_device(struct kvm *kvm, unsigned ioctl, 937 unsigned long arg) 938 { 939 return -ENOTTY; 940 } 941 942 #endif 943 944 static inline void kvm_make_request(int req, struct kvm_vcpu *vcpu) 945 { 946 set_bit(req, &vcpu->requests); 947 } 948 949 static inline bool kvm_check_request(int req, struct kvm_vcpu *vcpu) 950 { 951 if (test_bit(req, &vcpu->requests)) { 952 clear_bit(req, &vcpu->requests); 953 return true; 954 } else { 955 return false; 956 } 957 } 958 959 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT 960 961 static inline void kvm_vcpu_set_in_spin_loop(struct kvm_vcpu *vcpu, bool val) 962 { 963 vcpu->spin_loop.in_spin_loop = val; 964 } 965 static inline void kvm_vcpu_set_dy_eligible(struct kvm_vcpu *vcpu, bool val) 966 { 967 vcpu->spin_loop.dy_eligible = val; 968 } 969 970 #else /* !CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT */ 971 972 static inline void kvm_vcpu_set_in_spin_loop(struct kvm_vcpu *vcpu, bool val) 973 { 974 } 975 976 static inline void kvm_vcpu_set_dy_eligible(struct kvm_vcpu *vcpu, bool val) 977 { 978 } 979 980 static inline bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu) 981 { 982 return true; 983 } 984 985 #endif /* CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT */ 986 #endif 987 988