1 #ifndef __KVM_HOST_H 2 #define __KVM_HOST_H 3 4 /* 5 * This work is licensed under the terms of the GNU GPL, version 2. See 6 * the COPYING file in the top-level directory. 7 */ 8 9 #include <linux/types.h> 10 #include <linux/hardirq.h> 11 #include <linux/list.h> 12 #include <linux/mutex.h> 13 #include <linux/spinlock.h> 14 #include <linux/signal.h> 15 #include <linux/sched.h> 16 #include <linux/bug.h> 17 #include <linux/mm.h> 18 #include <linux/mmu_notifier.h> 19 #include <linux/preempt.h> 20 #include <linux/msi.h> 21 #include <linux/slab.h> 22 #include <linux/rcupdate.h> 23 #include <linux/ratelimit.h> 24 #include <linux/err.h> 25 #include <linux/irqflags.h> 26 #include <linux/context_tracking.h> 27 #include <linux/irqbypass.h> 28 #include <linux/swait.h> 29 #include <asm/signal.h> 30 31 #include <linux/kvm.h> 32 #include <linux/kvm_para.h> 33 34 #include <linux/kvm_types.h> 35 36 #include <asm/kvm_host.h> 37 38 #ifndef KVM_MAX_VCPU_ID 39 #define KVM_MAX_VCPU_ID KVM_MAX_VCPUS 40 #endif 41 42 /* 43 * The bit 16 ~ bit 31 of kvm_memory_region::flags are internally used 44 * in kvm, other bits are visible for userspace which are defined in 45 * include/linux/kvm_h. 46 */ 47 #define KVM_MEMSLOT_INVALID (1UL << 16) 48 #define KVM_MEMSLOT_INCOHERENT (1UL << 17) 49 50 /* Two fragments for cross MMIO pages. */ 51 #define KVM_MAX_MMIO_FRAGMENTS 2 52 53 #ifndef KVM_ADDRESS_SPACE_NUM 54 #define KVM_ADDRESS_SPACE_NUM 1 55 #endif 56 57 /* 58 * For the normal pfn, the highest 12 bits should be zero, 59 * so we can mask bit 62 ~ bit 52 to indicate the error pfn, 60 * mask bit 63 to indicate the noslot pfn. 61 */ 62 #define KVM_PFN_ERR_MASK (0x7ffULL << 52) 63 #define KVM_PFN_ERR_NOSLOT_MASK (0xfffULL << 52) 64 #define KVM_PFN_NOSLOT (0x1ULL << 63) 65 66 #define KVM_PFN_ERR_FAULT (KVM_PFN_ERR_MASK) 67 #define KVM_PFN_ERR_HWPOISON (KVM_PFN_ERR_MASK + 1) 68 #define KVM_PFN_ERR_RO_FAULT (KVM_PFN_ERR_MASK + 2) 69 70 /* 71 * error pfns indicate that the gfn is in slot but faild to 72 * translate it to pfn on host. 73 */ 74 static inline bool is_error_pfn(kvm_pfn_t pfn) 75 { 76 return !!(pfn & KVM_PFN_ERR_MASK); 77 } 78 79 /* 80 * error_noslot pfns indicate that the gfn can not be 81 * translated to pfn - it is not in slot or failed to 82 * translate it to pfn. 83 */ 84 static inline bool is_error_noslot_pfn(kvm_pfn_t pfn) 85 { 86 return !!(pfn & KVM_PFN_ERR_NOSLOT_MASK); 87 } 88 89 /* noslot pfn indicates that the gfn is not in slot. */ 90 static inline bool is_noslot_pfn(kvm_pfn_t pfn) 91 { 92 return pfn == KVM_PFN_NOSLOT; 93 } 94 95 /* 96 * architectures with KVM_HVA_ERR_BAD other than PAGE_OFFSET (e.g. s390) 97 * provide own defines and kvm_is_error_hva 98 */ 99 #ifndef KVM_HVA_ERR_BAD 100 101 #define KVM_HVA_ERR_BAD (PAGE_OFFSET) 102 #define KVM_HVA_ERR_RO_BAD (PAGE_OFFSET + PAGE_SIZE) 103 104 static inline bool kvm_is_error_hva(unsigned long addr) 105 { 106 return addr >= PAGE_OFFSET; 107 } 108 109 #endif 110 111 #define KVM_ERR_PTR_BAD_PAGE (ERR_PTR(-ENOENT)) 112 113 static inline bool is_error_page(struct page *page) 114 { 115 return IS_ERR(page); 116 } 117 118 /* 119 * Architecture-independent vcpu->requests bit members 120 * Bits 4-7 are reserved for more arch-independent bits. 121 */ 122 #define KVM_REQ_TLB_FLUSH 0 123 #define KVM_REQ_MMU_RELOAD 1 124 #define KVM_REQ_PENDING_TIMER 2 125 #define KVM_REQ_UNHALT 3 126 127 #define KVM_USERSPACE_IRQ_SOURCE_ID 0 128 #define KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID 1 129 130 extern struct kmem_cache *kvm_vcpu_cache; 131 132 extern spinlock_t kvm_lock; 133 extern struct list_head vm_list; 134 135 struct kvm_io_range { 136 gpa_t addr; 137 int len; 138 struct kvm_io_device *dev; 139 }; 140 141 #define NR_IOBUS_DEVS 1000 142 143 struct kvm_io_bus { 144 int dev_count; 145 int ioeventfd_count; 146 struct kvm_io_range range[]; 147 }; 148 149 enum kvm_bus { 150 KVM_MMIO_BUS, 151 KVM_PIO_BUS, 152 KVM_VIRTIO_CCW_NOTIFY_BUS, 153 KVM_FAST_MMIO_BUS, 154 KVM_NR_BUSES 155 }; 156 157 int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 158 int len, const void *val); 159 int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, 160 gpa_t addr, int len, const void *val, long cookie); 161 int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 162 int len, void *val); 163 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 164 int len, struct kvm_io_device *dev); 165 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, 166 struct kvm_io_device *dev); 167 168 #ifdef CONFIG_KVM_ASYNC_PF 169 struct kvm_async_pf { 170 struct work_struct work; 171 struct list_head link; 172 struct list_head queue; 173 struct kvm_vcpu *vcpu; 174 struct mm_struct *mm; 175 gva_t gva; 176 unsigned long addr; 177 struct kvm_arch_async_pf arch; 178 bool wakeup_all; 179 }; 180 181 void kvm_clear_async_pf_completion_queue(struct kvm_vcpu *vcpu); 182 void kvm_check_async_pf_completion(struct kvm_vcpu *vcpu); 183 int kvm_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, unsigned long hva, 184 struct kvm_arch_async_pf *arch); 185 int kvm_async_pf_wakeup_all(struct kvm_vcpu *vcpu); 186 #endif 187 188 enum { 189 OUTSIDE_GUEST_MODE, 190 IN_GUEST_MODE, 191 EXITING_GUEST_MODE, 192 READING_SHADOW_PAGE_TABLES, 193 }; 194 195 /* 196 * Sometimes a large or cross-page mmio needs to be broken up into separate 197 * exits for userspace servicing. 198 */ 199 struct kvm_mmio_fragment { 200 gpa_t gpa; 201 void *data; 202 unsigned len; 203 }; 204 205 struct kvm_vcpu { 206 struct kvm *kvm; 207 #ifdef CONFIG_PREEMPT_NOTIFIERS 208 struct preempt_notifier preempt_notifier; 209 #endif 210 int cpu; 211 int vcpu_id; 212 int srcu_idx; 213 int mode; 214 unsigned long requests; 215 unsigned long guest_debug; 216 217 int pre_pcpu; 218 struct list_head blocked_vcpu_list; 219 220 struct mutex mutex; 221 struct kvm_run *run; 222 223 int fpu_active; 224 int guest_fpu_loaded, guest_xcr0_loaded; 225 unsigned char fpu_counter; 226 struct swait_queue_head wq; 227 struct pid *pid; 228 int sigset_active; 229 sigset_t sigset; 230 struct kvm_vcpu_stat stat; 231 unsigned int halt_poll_ns; 232 bool valid_wakeup; 233 234 #ifdef CONFIG_HAS_IOMEM 235 int mmio_needed; 236 int mmio_read_completed; 237 int mmio_is_write; 238 int mmio_cur_fragment; 239 int mmio_nr_fragments; 240 struct kvm_mmio_fragment mmio_fragments[KVM_MAX_MMIO_FRAGMENTS]; 241 #endif 242 243 #ifdef CONFIG_KVM_ASYNC_PF 244 struct { 245 u32 queued; 246 struct list_head queue; 247 struct list_head done; 248 spinlock_t lock; 249 } async_pf; 250 #endif 251 252 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT 253 /* 254 * Cpu relax intercept or pause loop exit optimization 255 * in_spin_loop: set when a vcpu does a pause loop exit 256 * or cpu relax intercepted. 257 * dy_eligible: indicates whether vcpu is eligible for directed yield. 258 */ 259 struct { 260 bool in_spin_loop; 261 bool dy_eligible; 262 } spin_loop; 263 #endif 264 bool preempted; 265 struct kvm_vcpu_arch arch; 266 }; 267 268 static inline int kvm_vcpu_exiting_guest_mode(struct kvm_vcpu *vcpu) 269 { 270 return cmpxchg(&vcpu->mode, IN_GUEST_MODE, EXITING_GUEST_MODE); 271 } 272 273 /* 274 * Some of the bitops functions do not support too long bitmaps. 275 * This number must be determined not to exceed such limits. 276 */ 277 #define KVM_MEM_MAX_NR_PAGES ((1UL << 31) - 1) 278 279 struct kvm_memory_slot { 280 gfn_t base_gfn; 281 unsigned long npages; 282 unsigned long *dirty_bitmap; 283 struct kvm_arch_memory_slot arch; 284 unsigned long userspace_addr; 285 u32 flags; 286 short id; 287 }; 288 289 static inline unsigned long kvm_dirty_bitmap_bytes(struct kvm_memory_slot *memslot) 290 { 291 return ALIGN(memslot->npages, BITS_PER_LONG) / 8; 292 } 293 294 struct kvm_s390_adapter_int { 295 u64 ind_addr; 296 u64 summary_addr; 297 u64 ind_offset; 298 u32 summary_offset; 299 u32 adapter_id; 300 }; 301 302 struct kvm_hv_sint { 303 u32 vcpu; 304 u32 sint; 305 }; 306 307 struct kvm_kernel_irq_routing_entry { 308 u32 gsi; 309 u32 type; 310 int (*set)(struct kvm_kernel_irq_routing_entry *e, 311 struct kvm *kvm, int irq_source_id, int level, 312 bool line_status); 313 union { 314 struct { 315 unsigned irqchip; 316 unsigned pin; 317 } irqchip; 318 struct msi_msg msi; 319 struct kvm_s390_adapter_int adapter; 320 struct kvm_hv_sint hv_sint; 321 }; 322 struct hlist_node link; 323 }; 324 325 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 326 struct kvm_irq_routing_table { 327 int chip[KVM_NR_IRQCHIPS][KVM_IRQCHIP_NUM_PINS]; 328 u32 nr_rt_entries; 329 /* 330 * Array indexed by gsi. Each entry contains list of irq chips 331 * the gsi is connected to. 332 */ 333 struct hlist_head map[0]; 334 }; 335 #endif 336 337 #ifndef KVM_PRIVATE_MEM_SLOTS 338 #define KVM_PRIVATE_MEM_SLOTS 0 339 #endif 340 341 #ifndef KVM_MEM_SLOTS_NUM 342 #define KVM_MEM_SLOTS_NUM (KVM_USER_MEM_SLOTS + KVM_PRIVATE_MEM_SLOTS) 343 #endif 344 345 #ifndef __KVM_VCPU_MULTIPLE_ADDRESS_SPACE 346 static inline int kvm_arch_vcpu_memslots_id(struct kvm_vcpu *vcpu) 347 { 348 return 0; 349 } 350 #endif 351 352 /* 353 * Note: 354 * memslots are not sorted by id anymore, please use id_to_memslot() 355 * to get the memslot by its id. 356 */ 357 struct kvm_memslots { 358 u64 generation; 359 struct kvm_memory_slot memslots[KVM_MEM_SLOTS_NUM]; 360 /* The mapping table from slot id to the index in memslots[]. */ 361 short id_to_index[KVM_MEM_SLOTS_NUM]; 362 atomic_t lru_slot; 363 int used_slots; 364 }; 365 366 struct kvm { 367 spinlock_t mmu_lock; 368 struct mutex slots_lock; 369 struct mm_struct *mm; /* userspace tied to this vm */ 370 struct kvm_memslots *memslots[KVM_ADDRESS_SPACE_NUM]; 371 struct srcu_struct srcu; 372 struct srcu_struct irq_srcu; 373 struct kvm_vcpu *vcpus[KVM_MAX_VCPUS]; 374 atomic_t online_vcpus; 375 int last_boosted_vcpu; 376 struct list_head vm_list; 377 struct mutex lock; 378 struct kvm_io_bus *buses[KVM_NR_BUSES]; 379 #ifdef CONFIG_HAVE_KVM_EVENTFD 380 struct { 381 spinlock_t lock; 382 struct list_head items; 383 struct list_head resampler_list; 384 struct mutex resampler_lock; 385 } irqfds; 386 struct list_head ioeventfds; 387 #endif 388 struct kvm_vm_stat stat; 389 struct kvm_arch arch; 390 atomic_t users_count; 391 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET 392 struct kvm_coalesced_mmio_ring *coalesced_mmio_ring; 393 spinlock_t ring_lock; 394 struct list_head coalesced_zones; 395 #endif 396 397 struct mutex irq_lock; 398 #ifdef CONFIG_HAVE_KVM_IRQCHIP 399 /* 400 * Update side is protected by irq_lock. 401 */ 402 struct kvm_irq_routing_table __rcu *irq_routing; 403 #endif 404 #ifdef CONFIG_HAVE_KVM_IRQFD 405 struct hlist_head irq_ack_notifier_list; 406 #endif 407 408 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 409 struct mmu_notifier mmu_notifier; 410 unsigned long mmu_notifier_seq; 411 long mmu_notifier_count; 412 #endif 413 long tlbs_dirty; 414 struct list_head devices; 415 }; 416 417 #define kvm_err(fmt, ...) \ 418 pr_err("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__) 419 #define kvm_info(fmt, ...) \ 420 pr_info("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__) 421 #define kvm_debug(fmt, ...) \ 422 pr_debug("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__) 423 #define kvm_pr_unimpl(fmt, ...) \ 424 pr_err_ratelimited("kvm [%i]: " fmt, \ 425 task_tgid_nr(current), ## __VA_ARGS__) 426 427 /* The guest did something we don't support. */ 428 #define vcpu_unimpl(vcpu, fmt, ...) \ 429 kvm_pr_unimpl("vcpu%i, guest rIP: 0x%lx " fmt, \ 430 (vcpu)->vcpu_id, kvm_rip_read(vcpu), ## __VA_ARGS__) 431 432 #define vcpu_debug(vcpu, fmt, ...) \ 433 kvm_debug("vcpu%i " fmt, (vcpu)->vcpu_id, ## __VA_ARGS__) 434 #define vcpu_err(vcpu, fmt, ...) \ 435 kvm_err("vcpu%i " fmt, (vcpu)->vcpu_id, ## __VA_ARGS__) 436 437 static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i) 438 { 439 /* Pairs with smp_wmb() in kvm_vm_ioctl_create_vcpu, in case 440 * the caller has read kvm->online_vcpus before (as is the case 441 * for kvm_for_each_vcpu, for example). 442 */ 443 smp_rmb(); 444 return kvm->vcpus[i]; 445 } 446 447 #define kvm_for_each_vcpu(idx, vcpup, kvm) \ 448 for (idx = 0; \ 449 idx < atomic_read(&kvm->online_vcpus) && \ 450 (vcpup = kvm_get_vcpu(kvm, idx)) != NULL; \ 451 idx++) 452 453 static inline struct kvm_vcpu *kvm_get_vcpu_by_id(struct kvm *kvm, int id) 454 { 455 struct kvm_vcpu *vcpu = NULL; 456 int i; 457 458 if (id < 0) 459 return NULL; 460 if (id < KVM_MAX_VCPUS) 461 vcpu = kvm_get_vcpu(kvm, id); 462 if (vcpu && vcpu->vcpu_id == id) 463 return vcpu; 464 kvm_for_each_vcpu(i, vcpu, kvm) 465 if (vcpu->vcpu_id == id) 466 return vcpu; 467 return NULL; 468 } 469 470 #define kvm_for_each_memslot(memslot, slots) \ 471 for (memslot = &slots->memslots[0]; \ 472 memslot < slots->memslots + KVM_MEM_SLOTS_NUM && memslot->npages;\ 473 memslot++) 474 475 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id); 476 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu); 477 478 int __must_check vcpu_load(struct kvm_vcpu *vcpu); 479 void vcpu_put(struct kvm_vcpu *vcpu); 480 481 #ifdef __KVM_HAVE_IOAPIC 482 void kvm_vcpu_request_scan_ioapic(struct kvm *kvm); 483 void kvm_arch_post_irq_routing_update(struct kvm *kvm); 484 #else 485 static inline void kvm_vcpu_request_scan_ioapic(struct kvm *kvm) 486 { 487 } 488 static inline void kvm_arch_post_irq_routing_update(struct kvm *kvm) 489 { 490 } 491 #endif 492 493 #ifdef CONFIG_HAVE_KVM_IRQFD 494 int kvm_irqfd_init(void); 495 void kvm_irqfd_exit(void); 496 #else 497 static inline int kvm_irqfd_init(void) 498 { 499 return 0; 500 } 501 502 static inline void kvm_irqfd_exit(void) 503 { 504 } 505 #endif 506 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, 507 struct module *module); 508 void kvm_exit(void); 509 510 void kvm_get_kvm(struct kvm *kvm); 511 void kvm_put_kvm(struct kvm *kvm); 512 513 static inline struct kvm_memslots *__kvm_memslots(struct kvm *kvm, int as_id) 514 { 515 return rcu_dereference_check(kvm->memslots[as_id], 516 srcu_read_lock_held(&kvm->srcu) 517 || lockdep_is_held(&kvm->slots_lock)); 518 } 519 520 static inline struct kvm_memslots *kvm_memslots(struct kvm *kvm) 521 { 522 return __kvm_memslots(kvm, 0); 523 } 524 525 static inline struct kvm_memslots *kvm_vcpu_memslots(struct kvm_vcpu *vcpu) 526 { 527 int as_id = kvm_arch_vcpu_memslots_id(vcpu); 528 529 return __kvm_memslots(vcpu->kvm, as_id); 530 } 531 532 static inline struct kvm_memory_slot * 533 id_to_memslot(struct kvm_memslots *slots, int id) 534 { 535 int index = slots->id_to_index[id]; 536 struct kvm_memory_slot *slot; 537 538 slot = &slots->memslots[index]; 539 540 WARN_ON(slot->id != id); 541 return slot; 542 } 543 544 /* 545 * KVM_SET_USER_MEMORY_REGION ioctl allows the following operations: 546 * - create a new memory slot 547 * - delete an existing memory slot 548 * - modify an existing memory slot 549 * -- move it in the guest physical memory space 550 * -- just change its flags 551 * 552 * Since flags can be changed by some of these operations, the following 553 * differentiation is the best we can do for __kvm_set_memory_region(): 554 */ 555 enum kvm_mr_change { 556 KVM_MR_CREATE, 557 KVM_MR_DELETE, 558 KVM_MR_MOVE, 559 KVM_MR_FLAGS_ONLY, 560 }; 561 562 int kvm_set_memory_region(struct kvm *kvm, 563 const struct kvm_userspace_memory_region *mem); 564 int __kvm_set_memory_region(struct kvm *kvm, 565 const struct kvm_userspace_memory_region *mem); 566 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free, 567 struct kvm_memory_slot *dont); 568 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, 569 unsigned long npages); 570 void kvm_arch_memslots_updated(struct kvm *kvm, struct kvm_memslots *slots); 571 int kvm_arch_prepare_memory_region(struct kvm *kvm, 572 struct kvm_memory_slot *memslot, 573 const struct kvm_userspace_memory_region *mem, 574 enum kvm_mr_change change); 575 void kvm_arch_commit_memory_region(struct kvm *kvm, 576 const struct kvm_userspace_memory_region *mem, 577 const struct kvm_memory_slot *old, 578 const struct kvm_memory_slot *new, 579 enum kvm_mr_change change); 580 bool kvm_largepages_enabled(void); 581 void kvm_disable_largepages(void); 582 /* flush all memory translations */ 583 void kvm_arch_flush_shadow_all(struct kvm *kvm); 584 /* flush memory translations pointing to 'slot' */ 585 void kvm_arch_flush_shadow_memslot(struct kvm *kvm, 586 struct kvm_memory_slot *slot); 587 588 int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn, 589 struct page **pages, int nr_pages); 590 591 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn); 592 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn); 593 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable); 594 unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot, gfn_t gfn); 595 unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot, gfn_t gfn, 596 bool *writable); 597 void kvm_release_page_clean(struct page *page); 598 void kvm_release_page_dirty(struct page *page); 599 void kvm_set_page_accessed(struct page *page); 600 601 kvm_pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn); 602 kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn); 603 kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault, 604 bool *writable); 605 kvm_pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn); 606 kvm_pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn); 607 kvm_pfn_t __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, 608 bool atomic, bool *async, bool write_fault, 609 bool *writable); 610 611 void kvm_release_pfn_clean(kvm_pfn_t pfn); 612 void kvm_set_pfn_dirty(kvm_pfn_t pfn); 613 void kvm_set_pfn_accessed(kvm_pfn_t pfn); 614 void kvm_get_pfn(kvm_pfn_t pfn); 615 616 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, 617 int len); 618 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data, 619 unsigned long len); 620 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len); 621 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 622 void *data, unsigned long len); 623 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data, 624 int offset, int len); 625 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, 626 unsigned long len); 627 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 628 void *data, unsigned long len); 629 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 630 gpa_t gpa, unsigned long len); 631 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len); 632 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len); 633 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn); 634 bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn); 635 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn); 636 void mark_page_dirty(struct kvm *kvm, gfn_t gfn); 637 638 struct kvm_memslots *kvm_vcpu_memslots(struct kvm_vcpu *vcpu); 639 struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn); 640 kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn); 641 kvm_pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn); 642 struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn); 643 unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn); 644 unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable); 645 int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data, int offset, 646 int len); 647 int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, 648 unsigned long len); 649 int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, 650 unsigned long len); 651 int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, const void *data, 652 int offset, int len); 653 int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data, 654 unsigned long len); 655 void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn); 656 657 void kvm_vcpu_block(struct kvm_vcpu *vcpu); 658 void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu); 659 void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu); 660 void kvm_vcpu_wake_up(struct kvm_vcpu *vcpu); 661 void kvm_vcpu_kick(struct kvm_vcpu *vcpu); 662 int kvm_vcpu_yield_to(struct kvm_vcpu *target); 663 void kvm_vcpu_on_spin(struct kvm_vcpu *vcpu); 664 void kvm_load_guest_fpu(struct kvm_vcpu *vcpu); 665 void kvm_put_guest_fpu(struct kvm_vcpu *vcpu); 666 667 void kvm_flush_remote_tlbs(struct kvm *kvm); 668 void kvm_reload_remote_mmus(struct kvm *kvm); 669 bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req); 670 671 long kvm_arch_dev_ioctl(struct file *filp, 672 unsigned int ioctl, unsigned long arg); 673 long kvm_arch_vcpu_ioctl(struct file *filp, 674 unsigned int ioctl, unsigned long arg); 675 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf); 676 677 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext); 678 679 int kvm_get_dirty_log(struct kvm *kvm, 680 struct kvm_dirty_log *log, int *is_dirty); 681 682 int kvm_get_dirty_log_protect(struct kvm *kvm, 683 struct kvm_dirty_log *log, bool *is_dirty); 684 685 void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, 686 struct kvm_memory_slot *slot, 687 gfn_t gfn_offset, 688 unsigned long mask); 689 690 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, 691 struct kvm_dirty_log *log); 692 693 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, 694 bool line_status); 695 long kvm_arch_vm_ioctl(struct file *filp, 696 unsigned int ioctl, unsigned long arg); 697 698 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu); 699 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu); 700 701 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 702 struct kvm_translation *tr); 703 704 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs); 705 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs); 706 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, 707 struct kvm_sregs *sregs); 708 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, 709 struct kvm_sregs *sregs); 710 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 711 struct kvm_mp_state *mp_state); 712 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 713 struct kvm_mp_state *mp_state); 714 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 715 struct kvm_guest_debug *dbg); 716 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run); 717 718 int kvm_arch_init(void *opaque); 719 void kvm_arch_exit(void); 720 721 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu); 722 void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu); 723 724 void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu); 725 726 void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu); 727 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu); 728 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu); 729 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id); 730 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu); 731 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu); 732 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu); 733 734 int kvm_arch_hardware_enable(void); 735 void kvm_arch_hardware_disable(void); 736 int kvm_arch_hardware_setup(void); 737 void kvm_arch_hardware_unsetup(void); 738 void kvm_arch_check_processor_compat(void *rtn); 739 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu); 740 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu); 741 742 void *kvm_kvzalloc(unsigned long size); 743 744 #ifndef __KVM_HAVE_ARCH_VM_ALLOC 745 static inline struct kvm *kvm_arch_alloc_vm(void) 746 { 747 return kzalloc(sizeof(struct kvm), GFP_KERNEL); 748 } 749 750 static inline void kvm_arch_free_vm(struct kvm *kvm) 751 { 752 kfree(kvm); 753 } 754 #endif 755 756 #ifdef __KVM_HAVE_ARCH_NONCOHERENT_DMA 757 void kvm_arch_register_noncoherent_dma(struct kvm *kvm); 758 void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm); 759 bool kvm_arch_has_noncoherent_dma(struct kvm *kvm); 760 #else 761 static inline void kvm_arch_register_noncoherent_dma(struct kvm *kvm) 762 { 763 } 764 765 static inline void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm) 766 { 767 } 768 769 static inline bool kvm_arch_has_noncoherent_dma(struct kvm *kvm) 770 { 771 return false; 772 } 773 #endif 774 #ifdef __KVM_HAVE_ARCH_ASSIGNED_DEVICE 775 void kvm_arch_start_assignment(struct kvm *kvm); 776 void kvm_arch_end_assignment(struct kvm *kvm); 777 bool kvm_arch_has_assigned_device(struct kvm *kvm); 778 #else 779 static inline void kvm_arch_start_assignment(struct kvm *kvm) 780 { 781 } 782 783 static inline void kvm_arch_end_assignment(struct kvm *kvm) 784 { 785 } 786 787 static inline bool kvm_arch_has_assigned_device(struct kvm *kvm) 788 { 789 return false; 790 } 791 #endif 792 793 static inline struct swait_queue_head *kvm_arch_vcpu_wq(struct kvm_vcpu *vcpu) 794 { 795 #ifdef __KVM_HAVE_ARCH_WQP 796 return vcpu->arch.wqp; 797 #else 798 return &vcpu->wq; 799 #endif 800 } 801 802 #ifdef __KVM_HAVE_ARCH_INTC_INITIALIZED 803 /* 804 * returns true if the virtual interrupt controller is initialized and 805 * ready to accept virtual IRQ. On some architectures the virtual interrupt 806 * controller is dynamically instantiated and this is not always true. 807 */ 808 bool kvm_arch_intc_initialized(struct kvm *kvm); 809 #else 810 static inline bool kvm_arch_intc_initialized(struct kvm *kvm) 811 { 812 return true; 813 } 814 #endif 815 816 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type); 817 void kvm_arch_destroy_vm(struct kvm *kvm); 818 void kvm_arch_sync_events(struct kvm *kvm); 819 820 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu); 821 void kvm_vcpu_kick(struct kvm_vcpu *vcpu); 822 823 bool kvm_is_reserved_pfn(kvm_pfn_t pfn); 824 825 struct kvm_irq_ack_notifier { 826 struct hlist_node link; 827 unsigned gsi; 828 void (*irq_acked)(struct kvm_irq_ack_notifier *kian); 829 }; 830 831 int kvm_irq_map_gsi(struct kvm *kvm, 832 struct kvm_kernel_irq_routing_entry *entries, int gsi); 833 int kvm_irq_map_chip_pin(struct kvm *kvm, unsigned irqchip, unsigned pin); 834 835 int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level, 836 bool line_status); 837 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *irq_entry, struct kvm *kvm, 838 int irq_source_id, int level, bool line_status); 839 int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e, 840 struct kvm *kvm, int irq_source_id, 841 int level, bool line_status); 842 bool kvm_irq_has_notifier(struct kvm *kvm, unsigned irqchip, unsigned pin); 843 void kvm_notify_acked_gsi(struct kvm *kvm, int gsi); 844 void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin); 845 void kvm_register_irq_ack_notifier(struct kvm *kvm, 846 struct kvm_irq_ack_notifier *kian); 847 void kvm_unregister_irq_ack_notifier(struct kvm *kvm, 848 struct kvm_irq_ack_notifier *kian); 849 int kvm_request_irq_source_id(struct kvm *kvm); 850 void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id); 851 852 #ifdef CONFIG_KVM_DEVICE_ASSIGNMENT 853 int kvm_iommu_map_pages(struct kvm *kvm, struct kvm_memory_slot *slot); 854 void kvm_iommu_unmap_pages(struct kvm *kvm, struct kvm_memory_slot *slot); 855 #else 856 static inline int kvm_iommu_map_pages(struct kvm *kvm, 857 struct kvm_memory_slot *slot) 858 { 859 return 0; 860 } 861 862 static inline void kvm_iommu_unmap_pages(struct kvm *kvm, 863 struct kvm_memory_slot *slot) 864 { 865 } 866 #endif 867 868 /* must be called with irqs disabled */ 869 static inline void __kvm_guest_enter(void) 870 { 871 guest_enter(); 872 /* KVM does not hold any references to rcu protected data when it 873 * switches CPU into a guest mode. In fact switching to a guest mode 874 * is very similar to exiting to userspace from rcu point of view. In 875 * addition CPU may stay in a guest mode for quite a long time (up to 876 * one time slice). Lets treat guest mode as quiescent state, just like 877 * we do with user-mode execution. 878 */ 879 if (!context_tracking_cpu_is_enabled()) 880 rcu_virt_note_context_switch(smp_processor_id()); 881 } 882 883 /* must be called with irqs disabled */ 884 static inline void __kvm_guest_exit(void) 885 { 886 guest_exit(); 887 } 888 889 static inline void kvm_guest_enter(void) 890 { 891 unsigned long flags; 892 893 local_irq_save(flags); 894 __kvm_guest_enter(); 895 local_irq_restore(flags); 896 } 897 898 static inline void kvm_guest_exit(void) 899 { 900 unsigned long flags; 901 902 local_irq_save(flags); 903 __kvm_guest_exit(); 904 local_irq_restore(flags); 905 } 906 907 /* 908 * search_memslots() and __gfn_to_memslot() are here because they are 909 * used in non-modular code in arch/powerpc/kvm/book3s_hv_rm_mmu.c. 910 * gfn_to_memslot() itself isn't here as an inline because that would 911 * bloat other code too much. 912 */ 913 static inline struct kvm_memory_slot * 914 search_memslots(struct kvm_memslots *slots, gfn_t gfn) 915 { 916 int start = 0, end = slots->used_slots; 917 int slot = atomic_read(&slots->lru_slot); 918 struct kvm_memory_slot *memslots = slots->memslots; 919 920 if (gfn >= memslots[slot].base_gfn && 921 gfn < memslots[slot].base_gfn + memslots[slot].npages) 922 return &memslots[slot]; 923 924 while (start < end) { 925 slot = start + (end - start) / 2; 926 927 if (gfn >= memslots[slot].base_gfn) 928 end = slot; 929 else 930 start = slot + 1; 931 } 932 933 if (gfn >= memslots[start].base_gfn && 934 gfn < memslots[start].base_gfn + memslots[start].npages) { 935 atomic_set(&slots->lru_slot, start); 936 return &memslots[start]; 937 } 938 939 return NULL; 940 } 941 942 static inline struct kvm_memory_slot * 943 __gfn_to_memslot(struct kvm_memslots *slots, gfn_t gfn) 944 { 945 return search_memslots(slots, gfn); 946 } 947 948 static inline unsigned long 949 __gfn_to_hva_memslot(struct kvm_memory_slot *slot, gfn_t gfn) 950 { 951 return slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE; 952 } 953 954 static inline int memslot_id(struct kvm *kvm, gfn_t gfn) 955 { 956 return gfn_to_memslot(kvm, gfn)->id; 957 } 958 959 static inline gfn_t 960 hva_to_gfn_memslot(unsigned long hva, struct kvm_memory_slot *slot) 961 { 962 gfn_t gfn_offset = (hva - slot->userspace_addr) >> PAGE_SHIFT; 963 964 return slot->base_gfn + gfn_offset; 965 } 966 967 static inline gpa_t gfn_to_gpa(gfn_t gfn) 968 { 969 return (gpa_t)gfn << PAGE_SHIFT; 970 } 971 972 static inline gfn_t gpa_to_gfn(gpa_t gpa) 973 { 974 return (gfn_t)(gpa >> PAGE_SHIFT); 975 } 976 977 static inline hpa_t pfn_to_hpa(kvm_pfn_t pfn) 978 { 979 return (hpa_t)pfn << PAGE_SHIFT; 980 } 981 982 static inline bool kvm_is_error_gpa(struct kvm *kvm, gpa_t gpa) 983 { 984 unsigned long hva = gfn_to_hva(kvm, gpa_to_gfn(gpa)); 985 986 return kvm_is_error_hva(hva); 987 } 988 989 enum kvm_stat_kind { 990 KVM_STAT_VM, 991 KVM_STAT_VCPU, 992 }; 993 994 struct kvm_stats_debugfs_item { 995 const char *name; 996 int offset; 997 enum kvm_stat_kind kind; 998 }; 999 extern struct kvm_stats_debugfs_item debugfs_entries[]; 1000 extern struct dentry *kvm_debugfs_dir; 1001 1002 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) 1003 static inline int mmu_notifier_retry(struct kvm *kvm, unsigned long mmu_seq) 1004 { 1005 if (unlikely(kvm->mmu_notifier_count)) 1006 return 1; 1007 /* 1008 * Ensure the read of mmu_notifier_count happens before the read 1009 * of mmu_notifier_seq. This interacts with the smp_wmb() in 1010 * mmu_notifier_invalidate_range_end to make sure that the caller 1011 * either sees the old (non-zero) value of mmu_notifier_count or 1012 * the new (incremented) value of mmu_notifier_seq. 1013 * PowerPC Book3s HV KVM calls this under a per-page lock 1014 * rather than under kvm->mmu_lock, for scalability, so 1015 * can't rely on kvm->mmu_lock to keep things ordered. 1016 */ 1017 smp_rmb(); 1018 if (kvm->mmu_notifier_seq != mmu_seq) 1019 return 1; 1020 return 0; 1021 } 1022 #endif 1023 1024 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 1025 1026 #ifdef CONFIG_S390 1027 #define KVM_MAX_IRQ_ROUTES 4096 //FIXME: we can have more than that... 1028 #else 1029 #define KVM_MAX_IRQ_ROUTES 1024 1030 #endif 1031 1032 int kvm_setup_default_irq_routing(struct kvm *kvm); 1033 int kvm_setup_empty_irq_routing(struct kvm *kvm); 1034 int kvm_set_irq_routing(struct kvm *kvm, 1035 const struct kvm_irq_routing_entry *entries, 1036 unsigned nr, 1037 unsigned flags); 1038 int kvm_set_routing_entry(struct kvm_kernel_irq_routing_entry *e, 1039 const struct kvm_irq_routing_entry *ue); 1040 void kvm_free_irq_routing(struct kvm *kvm); 1041 1042 #else 1043 1044 static inline void kvm_free_irq_routing(struct kvm *kvm) {} 1045 1046 #endif 1047 1048 int kvm_send_userspace_msi(struct kvm *kvm, struct kvm_msi *msi); 1049 1050 #ifdef CONFIG_HAVE_KVM_EVENTFD 1051 1052 void kvm_eventfd_init(struct kvm *kvm); 1053 int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args); 1054 1055 #ifdef CONFIG_HAVE_KVM_IRQFD 1056 int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args); 1057 void kvm_irqfd_release(struct kvm *kvm); 1058 void kvm_irq_routing_update(struct kvm *); 1059 #else 1060 static inline int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args) 1061 { 1062 return -EINVAL; 1063 } 1064 1065 static inline void kvm_irqfd_release(struct kvm *kvm) {} 1066 #endif 1067 1068 #else 1069 1070 static inline void kvm_eventfd_init(struct kvm *kvm) {} 1071 1072 static inline int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args) 1073 { 1074 return -EINVAL; 1075 } 1076 1077 static inline void kvm_irqfd_release(struct kvm *kvm) {} 1078 1079 #ifdef CONFIG_HAVE_KVM_IRQCHIP 1080 static inline void kvm_irq_routing_update(struct kvm *kvm) 1081 { 1082 } 1083 #endif 1084 void kvm_arch_irq_routing_update(struct kvm *kvm); 1085 1086 static inline int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args) 1087 { 1088 return -ENOSYS; 1089 } 1090 1091 #endif /* CONFIG_HAVE_KVM_EVENTFD */ 1092 1093 #ifdef CONFIG_KVM_APIC_ARCHITECTURE 1094 bool kvm_vcpu_compatible(struct kvm_vcpu *vcpu); 1095 #else 1096 static inline bool kvm_vcpu_compatible(struct kvm_vcpu *vcpu) { return true; } 1097 #endif 1098 1099 static inline void kvm_make_request(int req, struct kvm_vcpu *vcpu) 1100 { 1101 /* 1102 * Ensure the rest of the request is published to kvm_check_request's 1103 * caller. Paired with the smp_mb__after_atomic in kvm_check_request. 1104 */ 1105 smp_wmb(); 1106 set_bit(req, &vcpu->requests); 1107 } 1108 1109 static inline bool kvm_check_request(int req, struct kvm_vcpu *vcpu) 1110 { 1111 if (test_bit(req, &vcpu->requests)) { 1112 clear_bit(req, &vcpu->requests); 1113 1114 /* 1115 * Ensure the rest of the request is visible to kvm_check_request's 1116 * caller. Paired with the smp_wmb in kvm_make_request. 1117 */ 1118 smp_mb__after_atomic(); 1119 return true; 1120 } else { 1121 return false; 1122 } 1123 } 1124 1125 extern bool kvm_rebooting; 1126 1127 struct kvm_device { 1128 struct kvm_device_ops *ops; 1129 struct kvm *kvm; 1130 void *private; 1131 struct list_head vm_node; 1132 }; 1133 1134 /* create, destroy, and name are mandatory */ 1135 struct kvm_device_ops { 1136 const char *name; 1137 int (*create)(struct kvm_device *dev, u32 type); 1138 1139 /* 1140 * Destroy is responsible for freeing dev. 1141 * 1142 * Destroy may be called before or after destructors are called 1143 * on emulated I/O regions, depending on whether a reference is 1144 * held by a vcpu or other kvm component that gets destroyed 1145 * after the emulated I/O. 1146 */ 1147 void (*destroy)(struct kvm_device *dev); 1148 1149 int (*set_attr)(struct kvm_device *dev, struct kvm_device_attr *attr); 1150 int (*get_attr)(struct kvm_device *dev, struct kvm_device_attr *attr); 1151 int (*has_attr)(struct kvm_device *dev, struct kvm_device_attr *attr); 1152 long (*ioctl)(struct kvm_device *dev, unsigned int ioctl, 1153 unsigned long arg); 1154 }; 1155 1156 void kvm_device_get(struct kvm_device *dev); 1157 void kvm_device_put(struct kvm_device *dev); 1158 struct kvm_device *kvm_device_from_filp(struct file *filp); 1159 int kvm_register_device_ops(struct kvm_device_ops *ops, u32 type); 1160 void kvm_unregister_device_ops(u32 type); 1161 1162 extern struct kvm_device_ops kvm_mpic_ops; 1163 extern struct kvm_device_ops kvm_xics_ops; 1164 extern struct kvm_device_ops kvm_arm_vgic_v2_ops; 1165 extern struct kvm_device_ops kvm_arm_vgic_v3_ops; 1166 1167 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT 1168 1169 static inline void kvm_vcpu_set_in_spin_loop(struct kvm_vcpu *vcpu, bool val) 1170 { 1171 vcpu->spin_loop.in_spin_loop = val; 1172 } 1173 static inline void kvm_vcpu_set_dy_eligible(struct kvm_vcpu *vcpu, bool val) 1174 { 1175 vcpu->spin_loop.dy_eligible = val; 1176 } 1177 1178 #else /* !CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT */ 1179 1180 static inline void kvm_vcpu_set_in_spin_loop(struct kvm_vcpu *vcpu, bool val) 1181 { 1182 } 1183 1184 static inline void kvm_vcpu_set_dy_eligible(struct kvm_vcpu *vcpu, bool val) 1185 { 1186 } 1187 #endif /* CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT */ 1188 1189 #ifdef CONFIG_HAVE_KVM_IRQ_BYPASS 1190 bool kvm_arch_has_irq_bypass(void); 1191 int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *, 1192 struct irq_bypass_producer *); 1193 void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *, 1194 struct irq_bypass_producer *); 1195 void kvm_arch_irq_bypass_stop(struct irq_bypass_consumer *); 1196 void kvm_arch_irq_bypass_start(struct irq_bypass_consumer *); 1197 int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq, 1198 uint32_t guest_irq, bool set); 1199 #endif /* CONFIG_HAVE_KVM_IRQ_BYPASS */ 1200 1201 #ifdef CONFIG_HAVE_KVM_INVALID_WAKEUPS 1202 /* If we wakeup during the poll time, was it a sucessful poll? */ 1203 static inline bool vcpu_valid_wakeup(struct kvm_vcpu *vcpu) 1204 { 1205 return vcpu->valid_wakeup; 1206 } 1207 1208 #else 1209 static inline bool vcpu_valid_wakeup(struct kvm_vcpu *vcpu) 1210 { 1211 return true; 1212 } 1213 #endif /* CONFIG_HAVE_KVM_INVALID_WAKEUPS */ 1214 1215 #endif 1216