1 /* 2 * Copyright 2008 Advanced Micro Devices, Inc. 3 * Copyright 2008 Red Hat Inc. 4 * Copyright 2009 Jerome Glisse. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: Dave Airlie 25 * Alex Deucher 26 * Jerome Glisse 27 */ 28 29 #include <linux/dma-fence-array.h> 30 #include <linux/interval_tree_generic.h> 31 #include <linux/idr.h> 32 #include <linux/dma-buf.h> 33 34 #include <drm/amdgpu_drm.h> 35 #include <drm/drm_drv.h> 36 #include <drm/ttm/ttm_tt.h> 37 #include <drm/drm_exec.h> 38 #include "amdgpu.h" 39 #include "amdgpu_vm.h" 40 #include "amdgpu_trace.h" 41 #include "amdgpu_amdkfd.h" 42 #include "amdgpu_gmc.h" 43 #include "amdgpu_xgmi.h" 44 #include "amdgpu_dma_buf.h" 45 #include "amdgpu_res_cursor.h" 46 #include "kfd_svm.h" 47 48 /** 49 * DOC: GPUVM 50 * 51 * GPUVM is the MMU functionality provided on the GPU. 52 * GPUVM is similar to the legacy GART on older asics, however 53 * rather than there being a single global GART table 54 * for the entire GPU, there can be multiple GPUVM page tables active 55 * at any given time. The GPUVM page tables can contain a mix 56 * VRAM pages and system pages (both memory and MMIO) and system pages 57 * can be mapped as snooped (cached system pages) or unsnooped 58 * (uncached system pages). 59 * 60 * Each active GPUVM has an ID associated with it and there is a page table 61 * linked with each VMID. When executing a command buffer, 62 * the kernel tells the engine what VMID to use for that command 63 * buffer. VMIDs are allocated dynamically as commands are submitted. 64 * The userspace drivers maintain their own address space and the kernel 65 * sets up their pages tables accordingly when they submit their 66 * command buffers and a VMID is assigned. 67 * The hardware supports up to 16 active GPUVMs at any given time. 68 * 69 * Each GPUVM is represented by a 1-2 or 1-5 level page table, depending 70 * on the ASIC family. GPUVM supports RWX attributes on each page as well 71 * as other features such as encryption and caching attributes. 72 * 73 * VMID 0 is special. It is the GPUVM used for the kernel driver. In 74 * addition to an aperture managed by a page table, VMID 0 also has 75 * several other apertures. There is an aperture for direct access to VRAM 76 * and there is a legacy AGP aperture which just forwards accesses directly 77 * to the matching system physical addresses (or IOVAs when an IOMMU is 78 * present). These apertures provide direct access to these memories without 79 * incurring the overhead of a page table. VMID 0 is used by the kernel 80 * driver for tasks like memory management. 81 * 82 * GPU clients (i.e., engines on the GPU) use GPUVM VMIDs to access memory. 83 * For user applications, each application can have their own unique GPUVM 84 * address space. The application manages the address space and the kernel 85 * driver manages the GPUVM page tables for each process. If an GPU client 86 * accesses an invalid page, it will generate a GPU page fault, similar to 87 * accessing an invalid page on a CPU. 88 */ 89 90 #define START(node) ((node)->start) 91 #define LAST(node) ((node)->last) 92 93 INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last, 94 START, LAST, static, amdgpu_vm_it) 95 96 #undef START 97 #undef LAST 98 99 /** 100 * struct amdgpu_prt_cb - Helper to disable partial resident texture feature from a fence callback 101 */ 102 struct amdgpu_prt_cb { 103 104 /** 105 * @adev: amdgpu device 106 */ 107 struct amdgpu_device *adev; 108 109 /** 110 * @cb: callback 111 */ 112 struct dma_fence_cb cb; 113 }; 114 115 /** 116 * struct amdgpu_vm_tlb_seq_struct - Helper to increment the TLB flush sequence 117 */ 118 struct amdgpu_vm_tlb_seq_struct { 119 /** 120 * @vm: pointer to the amdgpu_vm structure to set the fence sequence on 121 */ 122 struct amdgpu_vm *vm; 123 124 /** 125 * @cb: callback 126 */ 127 struct dma_fence_cb cb; 128 }; 129 130 /** 131 * amdgpu_vm_set_pasid - manage pasid and vm ptr mapping 132 * 133 * @adev: amdgpu_device pointer 134 * @vm: amdgpu_vm pointer 135 * @pasid: the pasid the VM is using on this GPU 136 * 137 * Set the pasid this VM is using on this GPU, can also be used to remove the 138 * pasid by passing in zero. 139 * 140 */ 141 int amdgpu_vm_set_pasid(struct amdgpu_device *adev, struct amdgpu_vm *vm, 142 u32 pasid) 143 { 144 int r; 145 146 if (vm->pasid == pasid) 147 return 0; 148 149 if (vm->pasid) { 150 r = xa_err(xa_erase_irq(&adev->vm_manager.pasids, vm->pasid)); 151 if (r < 0) 152 return r; 153 154 vm->pasid = 0; 155 } 156 157 if (pasid) { 158 r = xa_err(xa_store_irq(&adev->vm_manager.pasids, pasid, vm, 159 GFP_KERNEL)); 160 if (r < 0) 161 return r; 162 163 vm->pasid = pasid; 164 } 165 166 167 return 0; 168 } 169 170 /** 171 * amdgpu_vm_bo_evicted - vm_bo is evicted 172 * 173 * @vm_bo: vm_bo which is evicted 174 * 175 * State for PDs/PTs and per VM BOs which are not at the location they should 176 * be. 177 */ 178 static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo) 179 { 180 struct amdgpu_vm *vm = vm_bo->vm; 181 struct amdgpu_bo *bo = vm_bo->bo; 182 183 vm_bo->moved = true; 184 spin_lock(&vm_bo->vm->status_lock); 185 if (bo->tbo.type == ttm_bo_type_kernel) 186 list_move(&vm_bo->vm_status, &vm->evicted); 187 else 188 list_move_tail(&vm_bo->vm_status, &vm->evicted); 189 spin_unlock(&vm_bo->vm->status_lock); 190 } 191 /** 192 * amdgpu_vm_bo_moved - vm_bo is moved 193 * 194 * @vm_bo: vm_bo which is moved 195 * 196 * State for per VM BOs which are moved, but that change is not yet reflected 197 * in the page tables. 198 */ 199 static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo) 200 { 201 spin_lock(&vm_bo->vm->status_lock); 202 list_move(&vm_bo->vm_status, &vm_bo->vm->moved); 203 spin_unlock(&vm_bo->vm->status_lock); 204 } 205 206 /** 207 * amdgpu_vm_bo_idle - vm_bo is idle 208 * 209 * @vm_bo: vm_bo which is now idle 210 * 211 * State for PDs/PTs and per VM BOs which have gone through the state machine 212 * and are now idle. 213 */ 214 static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo) 215 { 216 spin_lock(&vm_bo->vm->status_lock); 217 list_move(&vm_bo->vm_status, &vm_bo->vm->idle); 218 spin_unlock(&vm_bo->vm->status_lock); 219 vm_bo->moved = false; 220 } 221 222 /** 223 * amdgpu_vm_bo_invalidated - vm_bo is invalidated 224 * 225 * @vm_bo: vm_bo which is now invalidated 226 * 227 * State for normal BOs which are invalidated and that change not yet reflected 228 * in the PTs. 229 */ 230 static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo) 231 { 232 spin_lock(&vm_bo->vm->status_lock); 233 list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated); 234 spin_unlock(&vm_bo->vm->status_lock); 235 } 236 237 /** 238 * amdgpu_vm_bo_evicted_user - vm_bo is evicted 239 * 240 * @vm_bo: vm_bo which is evicted 241 * 242 * State for BOs used by user mode queues which are not at the location they 243 * should be. 244 */ 245 static void amdgpu_vm_bo_evicted_user(struct amdgpu_vm_bo_base *vm_bo) 246 { 247 vm_bo->moved = true; 248 spin_lock(&vm_bo->vm->status_lock); 249 list_move(&vm_bo->vm_status, &vm_bo->vm->evicted_user); 250 spin_unlock(&vm_bo->vm->status_lock); 251 } 252 253 /** 254 * amdgpu_vm_bo_relocated - vm_bo is reloacted 255 * 256 * @vm_bo: vm_bo which is relocated 257 * 258 * State for PDs/PTs which needs to update their parent PD. 259 * For the root PD, just move to idle state. 260 */ 261 static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo) 262 { 263 if (vm_bo->bo->parent) { 264 spin_lock(&vm_bo->vm->status_lock); 265 list_move(&vm_bo->vm_status, &vm_bo->vm->relocated); 266 spin_unlock(&vm_bo->vm->status_lock); 267 } else { 268 amdgpu_vm_bo_idle(vm_bo); 269 } 270 } 271 272 /** 273 * amdgpu_vm_bo_done - vm_bo is done 274 * 275 * @vm_bo: vm_bo which is now done 276 * 277 * State for normal BOs which are invalidated and that change has been updated 278 * in the PTs. 279 */ 280 static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo) 281 { 282 spin_lock(&vm_bo->vm->status_lock); 283 list_move(&vm_bo->vm_status, &vm_bo->vm->done); 284 spin_unlock(&vm_bo->vm->status_lock); 285 } 286 287 /** 288 * amdgpu_vm_bo_reset_state_machine - reset the vm_bo state machine 289 * @vm: the VM which state machine to reset 290 * 291 * Move all vm_bo object in the VM into a state where they will be updated 292 * again during validation. 293 */ 294 static void amdgpu_vm_bo_reset_state_machine(struct amdgpu_vm *vm) 295 { 296 struct amdgpu_vm_bo_base *vm_bo, *tmp; 297 298 spin_lock(&vm->status_lock); 299 list_splice_init(&vm->done, &vm->invalidated); 300 list_for_each_entry(vm_bo, &vm->invalidated, vm_status) 301 vm_bo->moved = true; 302 list_for_each_entry_safe(vm_bo, tmp, &vm->idle, vm_status) { 303 struct amdgpu_bo *bo = vm_bo->bo; 304 305 vm_bo->moved = true; 306 if (!bo || bo->tbo.type != ttm_bo_type_kernel) 307 list_move(&vm_bo->vm_status, &vm_bo->vm->moved); 308 else if (bo->parent) 309 list_move(&vm_bo->vm_status, &vm_bo->vm->relocated); 310 } 311 spin_unlock(&vm->status_lock); 312 } 313 314 /** 315 * amdgpu_vm_update_shared - helper to update shared memory stat 316 * @base: base structure for tracking BO usage in a VM 317 * 318 * Takes the vm status_lock and updates the shared memory stat. If the basic 319 * stat changed (e.g. buffer was moved) amdgpu_vm_update_stats need to be called 320 * as well. 321 */ 322 static void amdgpu_vm_update_shared(struct amdgpu_vm_bo_base *base) 323 { 324 struct amdgpu_vm *vm = base->vm; 325 struct amdgpu_bo *bo = base->bo; 326 uint64_t size = amdgpu_bo_size(bo); 327 uint32_t bo_memtype = amdgpu_bo_mem_stats_placement(bo); 328 bool shared; 329 330 spin_lock(&vm->status_lock); 331 shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base); 332 if (base->shared != shared) { 333 base->shared = shared; 334 if (shared) { 335 vm->stats[bo_memtype].drm.shared += size; 336 vm->stats[bo_memtype].drm.private -= size; 337 } else { 338 vm->stats[bo_memtype].drm.shared -= size; 339 vm->stats[bo_memtype].drm.private += size; 340 } 341 } 342 spin_unlock(&vm->status_lock); 343 } 344 345 /** 346 * amdgpu_vm_bo_update_shared - callback when bo gets shared/unshared 347 * @bo: amdgpu buffer object 348 * 349 * Update the per VM stats for all the vm if needed from private to shared or 350 * vice versa. 351 */ 352 void amdgpu_vm_bo_update_shared(struct amdgpu_bo *bo) 353 { 354 struct amdgpu_vm_bo_base *base; 355 356 for (base = bo->vm_bo; base; base = base->next) 357 amdgpu_vm_update_shared(base); 358 } 359 360 /** 361 * amdgpu_vm_update_stats_locked - helper to update normal memory stat 362 * @base: base structure for tracking BO usage in a VM 363 * @res: the ttm_resource to use for the purpose of accounting, may or may not 364 * be bo->tbo.resource 365 * @sign: if we should add (+1) or subtract (-1) from the stat 366 * 367 * Caller need to have the vm status_lock held. Useful for when multiple update 368 * need to happen at the same time. 369 */ 370 static void amdgpu_vm_update_stats_locked(struct amdgpu_vm_bo_base *base, 371 struct ttm_resource *res, int sign) 372 { 373 struct amdgpu_vm *vm = base->vm; 374 struct amdgpu_bo *bo = base->bo; 375 int64_t size = sign * amdgpu_bo_size(bo); 376 uint32_t bo_memtype = amdgpu_bo_mem_stats_placement(bo); 377 378 /* For drm-total- and drm-shared-, BO are accounted by their preferred 379 * placement, see also amdgpu_bo_mem_stats_placement. 380 */ 381 if (base->shared) 382 vm->stats[bo_memtype].drm.shared += size; 383 else 384 vm->stats[bo_memtype].drm.private += size; 385 386 if (res && res->mem_type < __AMDGPU_PL_NUM) { 387 uint32_t res_memtype = res->mem_type; 388 389 vm->stats[res_memtype].drm.resident += size; 390 /* BO only count as purgeable if it is resident, 391 * since otherwise there's nothing to purge. 392 */ 393 if (bo->flags & AMDGPU_GEM_CREATE_DISCARDABLE) 394 vm->stats[res_memtype].drm.purgeable += size; 395 if (!(bo->preferred_domains & amdgpu_mem_type_to_domain(res_memtype))) 396 vm->stats[bo_memtype].evicted += size; 397 } 398 } 399 400 /** 401 * amdgpu_vm_update_stats - helper to update normal memory stat 402 * @base: base structure for tracking BO usage in a VM 403 * @res: the ttm_resource to use for the purpose of accounting, may or may not 404 * be bo->tbo.resource 405 * @sign: if we should add (+1) or subtract (-1) from the stat 406 * 407 * Updates the basic memory stat when bo is added/deleted/moved. 408 */ 409 void amdgpu_vm_update_stats(struct amdgpu_vm_bo_base *base, 410 struct ttm_resource *res, int sign) 411 { 412 struct amdgpu_vm *vm = base->vm; 413 414 spin_lock(&vm->status_lock); 415 amdgpu_vm_update_stats_locked(base, res, sign); 416 spin_unlock(&vm->status_lock); 417 } 418 419 /** 420 * amdgpu_vm_bo_base_init - Adds bo to the list of bos associated with the vm 421 * 422 * @base: base structure for tracking BO usage in a VM 423 * @vm: vm to which bo is to be added 424 * @bo: amdgpu buffer object 425 * 426 * Initialize a bo_va_base structure and add it to the appropriate lists 427 * 428 */ 429 void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base, 430 struct amdgpu_vm *vm, struct amdgpu_bo *bo) 431 { 432 base->vm = vm; 433 base->bo = bo; 434 base->next = NULL; 435 INIT_LIST_HEAD(&base->vm_status); 436 437 if (!bo) 438 return; 439 base->next = bo->vm_bo; 440 bo->vm_bo = base; 441 442 spin_lock(&vm->status_lock); 443 base->shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base); 444 amdgpu_vm_update_stats_locked(base, bo->tbo.resource, +1); 445 spin_unlock(&vm->status_lock); 446 447 if (!amdgpu_vm_is_bo_always_valid(vm, bo)) 448 return; 449 450 dma_resv_assert_held(vm->root.bo->tbo.base.resv); 451 452 ttm_bo_set_bulk_move(&bo->tbo, &vm->lru_bulk_move); 453 if (bo->tbo.type == ttm_bo_type_kernel && bo->parent) 454 amdgpu_vm_bo_relocated(base); 455 else 456 amdgpu_vm_bo_idle(base); 457 458 if (bo->preferred_domains & 459 amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type)) 460 return; 461 462 /* 463 * we checked all the prerequisites, but it looks like this per vm bo 464 * is currently evicted. add the bo to the evicted list to make sure it 465 * is validated on next vm use to avoid fault. 466 * */ 467 amdgpu_vm_bo_evicted(base); 468 } 469 470 /** 471 * amdgpu_vm_lock_pd - lock PD in drm_exec 472 * 473 * @vm: vm providing the BOs 474 * @exec: drm execution context 475 * @num_fences: number of extra fences to reserve 476 * 477 * Lock the VM root PD in the DRM execution context. 478 */ 479 int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct drm_exec *exec, 480 unsigned int num_fences) 481 { 482 /* We need at least two fences for the VM PD/PT updates */ 483 return drm_exec_prepare_obj(exec, &vm->root.bo->tbo.base, 484 2 + num_fences); 485 } 486 487 /** 488 * amdgpu_vm_move_to_lru_tail - move all BOs to the end of LRU 489 * 490 * @adev: amdgpu device pointer 491 * @vm: vm providing the BOs 492 * 493 * Move all BOs to the end of LRU and remember their positions to put them 494 * together. 495 */ 496 void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev, 497 struct amdgpu_vm *vm) 498 { 499 spin_lock(&adev->mman.bdev.lru_lock); 500 ttm_lru_bulk_move_tail(&vm->lru_bulk_move); 501 spin_unlock(&adev->mman.bdev.lru_lock); 502 } 503 504 /* Create scheduler entities for page table updates */ 505 static int amdgpu_vm_init_entities(struct amdgpu_device *adev, 506 struct amdgpu_vm *vm) 507 { 508 int r; 509 510 r = drm_sched_entity_init(&vm->immediate, DRM_SCHED_PRIORITY_NORMAL, 511 adev->vm_manager.vm_pte_scheds, 512 adev->vm_manager.vm_pte_num_scheds, NULL); 513 if (r) 514 goto error; 515 516 return drm_sched_entity_init(&vm->delayed, DRM_SCHED_PRIORITY_NORMAL, 517 adev->vm_manager.vm_pte_scheds, 518 adev->vm_manager.vm_pte_num_scheds, NULL); 519 520 error: 521 drm_sched_entity_destroy(&vm->immediate); 522 return r; 523 } 524 525 /* Destroy the entities for page table updates again */ 526 static void amdgpu_vm_fini_entities(struct amdgpu_vm *vm) 527 { 528 drm_sched_entity_destroy(&vm->immediate); 529 drm_sched_entity_destroy(&vm->delayed); 530 } 531 532 /** 533 * amdgpu_vm_generation - return the page table re-generation counter 534 * @adev: the amdgpu_device 535 * @vm: optional VM to check, might be NULL 536 * 537 * Returns a page table re-generation token to allow checking if submissions 538 * are still valid to use this VM. The VM parameter might be NULL in which case 539 * just the VRAM lost counter will be used. 540 */ 541 uint64_t amdgpu_vm_generation(struct amdgpu_device *adev, struct amdgpu_vm *vm) 542 { 543 uint64_t result = (u64)atomic_read(&adev->vram_lost_counter) << 32; 544 545 if (!vm) 546 return result; 547 548 result += lower_32_bits(vm->generation); 549 /* Add one if the page tables will be re-generated on next CS */ 550 if (drm_sched_entity_error(&vm->delayed)) 551 ++result; 552 553 return result; 554 } 555 556 /** 557 * amdgpu_vm_validate - validate evicted BOs tracked in the VM 558 * 559 * @adev: amdgpu device pointer 560 * @vm: vm providing the BOs 561 * @ticket: optional reservation ticket used to reserve the VM 562 * @validate: callback to do the validation 563 * @param: parameter for the validation callback 564 * 565 * Validate the page table BOs and per-VM BOs on command submission if 566 * necessary. If a ticket is given, also try to validate evicted user queue 567 * BOs. They must already be reserved with the given ticket. 568 * 569 * Returns: 570 * Validation result. 571 */ 572 int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm, 573 struct ww_acquire_ctx *ticket, 574 int (*validate)(void *p, struct amdgpu_bo *bo), 575 void *param) 576 { 577 uint64_t new_vm_generation = amdgpu_vm_generation(adev, vm); 578 struct amdgpu_vm_bo_base *bo_base; 579 struct amdgpu_bo *bo; 580 int r; 581 582 if (vm->generation != new_vm_generation) { 583 vm->generation = new_vm_generation; 584 amdgpu_vm_bo_reset_state_machine(vm); 585 amdgpu_vm_fini_entities(vm); 586 r = amdgpu_vm_init_entities(adev, vm); 587 if (r) 588 return r; 589 } 590 591 spin_lock(&vm->status_lock); 592 while (!list_empty(&vm->evicted)) { 593 bo_base = list_first_entry(&vm->evicted, 594 struct amdgpu_vm_bo_base, 595 vm_status); 596 spin_unlock(&vm->status_lock); 597 598 bo = bo_base->bo; 599 600 r = validate(param, bo); 601 if (r) 602 return r; 603 604 if (bo->tbo.type != ttm_bo_type_kernel) { 605 amdgpu_vm_bo_moved(bo_base); 606 } else { 607 vm->update_funcs->map_table(to_amdgpu_bo_vm(bo)); 608 amdgpu_vm_bo_relocated(bo_base); 609 } 610 spin_lock(&vm->status_lock); 611 } 612 while (ticket && !list_empty(&vm->evicted_user)) { 613 bo_base = list_first_entry(&vm->evicted_user, 614 struct amdgpu_vm_bo_base, 615 vm_status); 616 spin_unlock(&vm->status_lock); 617 618 bo = bo_base->bo; 619 620 if (dma_resv_locking_ctx(bo->tbo.base.resv) != ticket) { 621 struct amdgpu_task_info *ti = amdgpu_vm_get_task_info_vm(vm); 622 623 pr_warn_ratelimited("Evicted user BO is not reserved\n"); 624 if (ti) { 625 pr_warn_ratelimited("pid %d\n", ti->pid); 626 amdgpu_vm_put_task_info(ti); 627 } 628 629 return -EINVAL; 630 } 631 632 r = validate(param, bo); 633 if (r) 634 return r; 635 636 amdgpu_vm_bo_invalidated(bo_base); 637 638 spin_lock(&vm->status_lock); 639 } 640 spin_unlock(&vm->status_lock); 641 642 amdgpu_vm_eviction_lock(vm); 643 vm->evicting = false; 644 amdgpu_vm_eviction_unlock(vm); 645 646 return 0; 647 } 648 649 /** 650 * amdgpu_vm_ready - check VM is ready for updates 651 * 652 * @vm: VM to check 653 * 654 * Check if all VM PDs/PTs are ready for updates 655 * 656 * Returns: 657 * True if VM is not evicting. 658 */ 659 bool amdgpu_vm_ready(struct amdgpu_vm *vm) 660 { 661 bool empty; 662 bool ret; 663 664 amdgpu_vm_eviction_lock(vm); 665 ret = !vm->evicting; 666 amdgpu_vm_eviction_unlock(vm); 667 668 spin_lock(&vm->status_lock); 669 empty = list_empty(&vm->evicted); 670 spin_unlock(&vm->status_lock); 671 672 return ret && empty; 673 } 674 675 /** 676 * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug 677 * 678 * @adev: amdgpu_device pointer 679 */ 680 void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev) 681 { 682 const struct amdgpu_ip_block *ip_block; 683 bool has_compute_vm_bug; 684 struct amdgpu_ring *ring; 685 int i; 686 687 has_compute_vm_bug = false; 688 689 ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX); 690 if (ip_block) { 691 /* Compute has a VM bug for GFX version < 7. 692 Compute has a VM bug for GFX 8 MEC firmware version < 673.*/ 693 if (ip_block->version->major <= 7) 694 has_compute_vm_bug = true; 695 else if (ip_block->version->major == 8) 696 if (adev->gfx.mec_fw_version < 673) 697 has_compute_vm_bug = true; 698 } 699 700 for (i = 0; i < adev->num_rings; i++) { 701 ring = adev->rings[i]; 702 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) 703 /* only compute rings */ 704 ring->has_compute_vm_bug = has_compute_vm_bug; 705 else 706 ring->has_compute_vm_bug = false; 707 } 708 } 709 710 /** 711 * amdgpu_vm_need_pipeline_sync - Check if pipe sync is needed for job. 712 * 713 * @ring: ring on which the job will be submitted 714 * @job: job to submit 715 * 716 * Returns: 717 * True if sync is needed. 718 */ 719 bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring, 720 struct amdgpu_job *job) 721 { 722 struct amdgpu_device *adev = ring->adev; 723 unsigned vmhub = ring->vm_hub; 724 struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; 725 726 if (job->vmid == 0) 727 return false; 728 729 if (job->vm_needs_flush || ring->has_compute_vm_bug) 730 return true; 731 732 if (ring->funcs->emit_gds_switch && job->gds_switch_needed) 733 return true; 734 735 if (amdgpu_vmid_had_gpu_reset(adev, &id_mgr->ids[job->vmid])) 736 return true; 737 738 return false; 739 } 740 741 /** 742 * amdgpu_vm_flush - hardware flush the vm 743 * 744 * @ring: ring to use for flush 745 * @job: related job 746 * @need_pipe_sync: is pipe sync needed 747 * 748 * Emit a VM flush when it is necessary. 749 * 750 * Returns: 751 * 0 on success, errno otherwise. 752 */ 753 int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, 754 bool need_pipe_sync) 755 { 756 struct amdgpu_device *adev = ring->adev; 757 struct amdgpu_isolation *isolation = &adev->isolation[ring->xcp_id]; 758 unsigned vmhub = ring->vm_hub; 759 struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; 760 struct amdgpu_vmid *id = &id_mgr->ids[job->vmid]; 761 bool spm_update_needed = job->spm_update_needed; 762 bool gds_switch_needed = ring->funcs->emit_gds_switch && 763 job->gds_switch_needed; 764 bool vm_flush_needed = job->vm_needs_flush; 765 bool cleaner_shader_needed = false; 766 bool pasid_mapping_needed = false; 767 struct dma_fence *fence = NULL; 768 unsigned int patch; 769 int r; 770 771 if (amdgpu_vmid_had_gpu_reset(adev, id)) { 772 gds_switch_needed = true; 773 vm_flush_needed = true; 774 pasid_mapping_needed = true; 775 spm_update_needed = true; 776 } 777 778 mutex_lock(&id_mgr->lock); 779 if (id->pasid != job->pasid || !id->pasid_mapping || 780 !dma_fence_is_signaled(id->pasid_mapping)) 781 pasid_mapping_needed = true; 782 mutex_unlock(&id_mgr->lock); 783 784 gds_switch_needed &= !!ring->funcs->emit_gds_switch; 785 vm_flush_needed &= !!ring->funcs->emit_vm_flush && 786 job->vm_pd_addr != AMDGPU_BO_INVALID_OFFSET; 787 pasid_mapping_needed &= adev->gmc.gmc_funcs->emit_pasid_mapping && 788 ring->funcs->emit_wreg; 789 790 cleaner_shader_needed = adev->gfx.enable_cleaner_shader && 791 ring->funcs->emit_cleaner_shader && job->base.s_fence && 792 &job->base.s_fence->scheduled == isolation->spearhead; 793 794 if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync && 795 !cleaner_shader_needed) 796 return 0; 797 798 amdgpu_ring_ib_begin(ring); 799 if (ring->funcs->init_cond_exec) 800 patch = amdgpu_ring_init_cond_exec(ring, 801 ring->cond_exe_gpu_addr); 802 803 if (need_pipe_sync) 804 amdgpu_ring_emit_pipeline_sync(ring); 805 806 if (cleaner_shader_needed) 807 ring->funcs->emit_cleaner_shader(ring); 808 809 if (vm_flush_needed) { 810 trace_amdgpu_vm_flush(ring, job->vmid, job->vm_pd_addr); 811 amdgpu_ring_emit_vm_flush(ring, job->vmid, job->vm_pd_addr); 812 } 813 814 if (pasid_mapping_needed) 815 amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid); 816 817 if (spm_update_needed && adev->gfx.rlc.funcs->update_spm_vmid) 818 adev->gfx.rlc.funcs->update_spm_vmid(adev, ring, job->vmid); 819 820 if (!ring->is_mes_queue && ring->funcs->emit_gds_switch && 821 gds_switch_needed) { 822 amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base, 823 job->gds_size, job->gws_base, 824 job->gws_size, job->oa_base, 825 job->oa_size); 826 } 827 828 if (vm_flush_needed || pasid_mapping_needed || cleaner_shader_needed) { 829 r = amdgpu_fence_emit(ring, &fence, NULL, 0); 830 if (r) 831 return r; 832 } 833 834 if (vm_flush_needed) { 835 mutex_lock(&id_mgr->lock); 836 dma_fence_put(id->last_flush); 837 id->last_flush = dma_fence_get(fence); 838 id->current_gpu_reset_count = 839 atomic_read(&adev->gpu_reset_counter); 840 mutex_unlock(&id_mgr->lock); 841 } 842 843 if (pasid_mapping_needed) { 844 mutex_lock(&id_mgr->lock); 845 id->pasid = job->pasid; 846 dma_fence_put(id->pasid_mapping); 847 id->pasid_mapping = dma_fence_get(fence); 848 mutex_unlock(&id_mgr->lock); 849 } 850 851 /* 852 * Make sure that all other submissions wait for the cleaner shader to 853 * finish before we push them to the HW. 854 */ 855 if (cleaner_shader_needed) { 856 mutex_lock(&adev->enforce_isolation_mutex); 857 dma_fence_put(isolation->spearhead); 858 isolation->spearhead = dma_fence_get(fence); 859 mutex_unlock(&adev->enforce_isolation_mutex); 860 } 861 dma_fence_put(fence); 862 863 amdgpu_ring_patch_cond_exec(ring, patch); 864 865 /* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */ 866 if (ring->funcs->emit_switch_buffer) { 867 amdgpu_ring_emit_switch_buffer(ring); 868 amdgpu_ring_emit_switch_buffer(ring); 869 } 870 871 amdgpu_ring_ib_end(ring); 872 return 0; 873 } 874 875 /** 876 * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo 877 * 878 * @vm: requested vm 879 * @bo: requested buffer object 880 * 881 * Find @bo inside the requested vm. 882 * Search inside the @bos vm list for the requested vm 883 * Returns the found bo_va or NULL if none is found 884 * 885 * Object has to be reserved! 886 * 887 * Returns: 888 * Found bo_va or NULL. 889 */ 890 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm, 891 struct amdgpu_bo *bo) 892 { 893 struct amdgpu_vm_bo_base *base; 894 895 for (base = bo->vm_bo; base; base = base->next) { 896 if (base->vm != vm) 897 continue; 898 899 return container_of(base, struct amdgpu_bo_va, base); 900 } 901 return NULL; 902 } 903 904 /** 905 * amdgpu_vm_map_gart - Resolve gart mapping of addr 906 * 907 * @pages_addr: optional DMA address to use for lookup 908 * @addr: the unmapped addr 909 * 910 * Look up the physical address of the page that the pte resolves 911 * to. 912 * 913 * Returns: 914 * The pointer for the page table entry. 915 */ 916 uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr) 917 { 918 uint64_t result; 919 920 /* page table offset */ 921 result = pages_addr[addr >> PAGE_SHIFT]; 922 923 /* in case cpu page size != gpu page size*/ 924 result |= addr & (~PAGE_MASK); 925 926 result &= 0xFFFFFFFFFFFFF000ULL; 927 928 return result; 929 } 930 931 /** 932 * amdgpu_vm_update_pdes - make sure that all directories are valid 933 * 934 * @adev: amdgpu_device pointer 935 * @vm: requested vm 936 * @immediate: submit immediately to the paging queue 937 * 938 * Makes sure all directories are up to date. 939 * 940 * Returns: 941 * 0 for success, error for failure. 942 */ 943 int amdgpu_vm_update_pdes(struct amdgpu_device *adev, 944 struct amdgpu_vm *vm, bool immediate) 945 { 946 struct amdgpu_vm_update_params params; 947 struct amdgpu_vm_bo_base *entry; 948 bool flush_tlb_needed = false; 949 LIST_HEAD(relocated); 950 int r, idx; 951 952 spin_lock(&vm->status_lock); 953 list_splice_init(&vm->relocated, &relocated); 954 spin_unlock(&vm->status_lock); 955 956 if (list_empty(&relocated)) 957 return 0; 958 959 if (!drm_dev_enter(adev_to_drm(adev), &idx)) 960 return -ENODEV; 961 962 memset(¶ms, 0, sizeof(params)); 963 params.adev = adev; 964 params.vm = vm; 965 params.immediate = immediate; 966 967 r = vm->update_funcs->prepare(¶ms, NULL); 968 if (r) 969 goto error; 970 971 list_for_each_entry(entry, &relocated, vm_status) { 972 /* vm_flush_needed after updating moved PDEs */ 973 flush_tlb_needed |= entry->moved; 974 975 r = amdgpu_vm_pde_update(¶ms, entry); 976 if (r) 977 goto error; 978 } 979 980 r = vm->update_funcs->commit(¶ms, &vm->last_update); 981 if (r) 982 goto error; 983 984 if (flush_tlb_needed) 985 atomic64_inc(&vm->tlb_seq); 986 987 while (!list_empty(&relocated)) { 988 entry = list_first_entry(&relocated, struct amdgpu_vm_bo_base, 989 vm_status); 990 amdgpu_vm_bo_idle(entry); 991 } 992 993 error: 994 drm_dev_exit(idx); 995 return r; 996 } 997 998 /** 999 * amdgpu_vm_tlb_seq_cb - make sure to increment tlb sequence 1000 * @fence: unused 1001 * @cb: the callback structure 1002 * 1003 * Increments the tlb sequence to make sure that future CS execute a VM flush. 1004 */ 1005 static void amdgpu_vm_tlb_seq_cb(struct dma_fence *fence, 1006 struct dma_fence_cb *cb) 1007 { 1008 struct amdgpu_vm_tlb_seq_struct *tlb_cb; 1009 1010 tlb_cb = container_of(cb, typeof(*tlb_cb), cb); 1011 atomic64_inc(&tlb_cb->vm->tlb_seq); 1012 kfree(tlb_cb); 1013 } 1014 1015 /** 1016 * amdgpu_vm_tlb_flush - prepare TLB flush 1017 * 1018 * @params: parameters for update 1019 * @fence: input fence to sync TLB flush with 1020 * @tlb_cb: the callback structure 1021 * 1022 * Increments the tlb sequence to make sure that future CS execute a VM flush. 1023 */ 1024 static void 1025 amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, 1026 struct dma_fence **fence, 1027 struct amdgpu_vm_tlb_seq_struct *tlb_cb) 1028 { 1029 struct amdgpu_vm *vm = params->vm; 1030 1031 tlb_cb->vm = vm; 1032 if (!fence || !*fence) { 1033 amdgpu_vm_tlb_seq_cb(NULL, &tlb_cb->cb); 1034 return; 1035 } 1036 1037 if (!dma_fence_add_callback(*fence, &tlb_cb->cb, 1038 amdgpu_vm_tlb_seq_cb)) { 1039 dma_fence_put(vm->last_tlb_flush); 1040 vm->last_tlb_flush = dma_fence_get(*fence); 1041 } else { 1042 amdgpu_vm_tlb_seq_cb(NULL, &tlb_cb->cb); 1043 } 1044 1045 /* Prepare a TLB flush fence to be attached to PTs */ 1046 if (!params->unlocked && vm->is_compute_context) { 1047 amdgpu_vm_tlb_fence_create(params->adev, vm, fence); 1048 1049 /* Makes sure no PD/PT is freed before the flush */ 1050 dma_resv_add_fence(vm->root.bo->tbo.base.resv, *fence, 1051 DMA_RESV_USAGE_BOOKKEEP); 1052 } 1053 } 1054 1055 /** 1056 * amdgpu_vm_update_range - update a range in the vm page table 1057 * 1058 * @adev: amdgpu_device pointer to use for commands 1059 * @vm: the VM to update the range 1060 * @immediate: immediate submission in a page fault 1061 * @unlocked: unlocked invalidation during MM callback 1062 * @flush_tlb: trigger tlb invalidation after update completed 1063 * @allow_override: change MTYPE for local NUMA nodes 1064 * @sync: fences we need to sync to 1065 * @start: start of mapped range 1066 * @last: last mapped entry 1067 * @flags: flags for the entries 1068 * @offset: offset into nodes and pages_addr 1069 * @vram_base: base for vram mappings 1070 * @res: ttm_resource to map 1071 * @pages_addr: DMA addresses to use for mapping 1072 * @fence: optional resulting fence 1073 * 1074 * Fill in the page table entries between @start and @last. 1075 * 1076 * Returns: 1077 * 0 for success, negative erro code for failure. 1078 */ 1079 int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm, 1080 bool immediate, bool unlocked, bool flush_tlb, 1081 bool allow_override, struct amdgpu_sync *sync, 1082 uint64_t start, uint64_t last, uint64_t flags, 1083 uint64_t offset, uint64_t vram_base, 1084 struct ttm_resource *res, dma_addr_t *pages_addr, 1085 struct dma_fence **fence) 1086 { 1087 struct amdgpu_vm_tlb_seq_struct *tlb_cb; 1088 struct amdgpu_vm_update_params params; 1089 struct amdgpu_res_cursor cursor; 1090 int r, idx; 1091 1092 if (!drm_dev_enter(adev_to_drm(adev), &idx)) 1093 return -ENODEV; 1094 1095 tlb_cb = kmalloc(sizeof(*tlb_cb), GFP_KERNEL); 1096 if (!tlb_cb) { 1097 drm_dev_exit(idx); 1098 return -ENOMEM; 1099 } 1100 1101 /* Vega20+XGMI where PTEs get inadvertently cached in L2 texture cache, 1102 * heavy-weight flush TLB unconditionally. 1103 */ 1104 flush_tlb |= adev->gmc.xgmi.num_physical_nodes && 1105 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 0); 1106 1107 /* 1108 * On GFX8 and older any 8 PTE block with a valid bit set enters the TLB 1109 */ 1110 flush_tlb |= amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(9, 0, 0); 1111 1112 memset(¶ms, 0, sizeof(params)); 1113 params.adev = adev; 1114 params.vm = vm; 1115 params.immediate = immediate; 1116 params.pages_addr = pages_addr; 1117 params.unlocked = unlocked; 1118 params.needs_flush = flush_tlb; 1119 params.allow_override = allow_override; 1120 INIT_LIST_HEAD(¶ms.tlb_flush_waitlist); 1121 1122 amdgpu_vm_eviction_lock(vm); 1123 if (vm->evicting) { 1124 r = -EBUSY; 1125 goto error_free; 1126 } 1127 1128 if (!unlocked && !dma_fence_is_signaled(vm->last_unlocked)) { 1129 struct dma_fence *tmp = dma_fence_get_stub(); 1130 1131 amdgpu_bo_fence(vm->root.bo, vm->last_unlocked, true); 1132 swap(vm->last_unlocked, tmp); 1133 dma_fence_put(tmp); 1134 } 1135 1136 r = vm->update_funcs->prepare(¶ms, sync); 1137 if (r) 1138 goto error_free; 1139 1140 amdgpu_res_first(pages_addr ? NULL : res, offset, 1141 (last - start + 1) * AMDGPU_GPU_PAGE_SIZE, &cursor); 1142 while (cursor.remaining) { 1143 uint64_t tmp, num_entries, addr; 1144 1145 num_entries = cursor.size >> AMDGPU_GPU_PAGE_SHIFT; 1146 if (pages_addr) { 1147 bool contiguous = true; 1148 1149 if (num_entries > AMDGPU_GPU_PAGES_IN_CPU_PAGE) { 1150 uint64_t pfn = cursor.start >> PAGE_SHIFT; 1151 uint64_t count; 1152 1153 contiguous = pages_addr[pfn + 1] == 1154 pages_addr[pfn] + PAGE_SIZE; 1155 1156 tmp = num_entries / 1157 AMDGPU_GPU_PAGES_IN_CPU_PAGE; 1158 for (count = 2; count < tmp; ++count) { 1159 uint64_t idx = pfn + count; 1160 1161 if (contiguous != (pages_addr[idx] == 1162 pages_addr[idx - 1] + PAGE_SIZE)) 1163 break; 1164 } 1165 if (!contiguous) 1166 count--; 1167 num_entries = count * 1168 AMDGPU_GPU_PAGES_IN_CPU_PAGE; 1169 } 1170 1171 if (!contiguous) { 1172 addr = cursor.start; 1173 params.pages_addr = pages_addr; 1174 } else { 1175 addr = pages_addr[cursor.start >> PAGE_SHIFT]; 1176 params.pages_addr = NULL; 1177 } 1178 1179 } else if (flags & (AMDGPU_PTE_VALID | AMDGPU_PTE_PRT_FLAG(adev))) { 1180 addr = vram_base + cursor.start; 1181 } else { 1182 addr = 0; 1183 } 1184 1185 tmp = start + num_entries; 1186 r = amdgpu_vm_ptes_update(¶ms, start, tmp, addr, flags); 1187 if (r) 1188 goto error_free; 1189 1190 amdgpu_res_next(&cursor, num_entries * AMDGPU_GPU_PAGE_SIZE); 1191 start = tmp; 1192 } 1193 1194 r = vm->update_funcs->commit(¶ms, fence); 1195 if (r) 1196 goto error_free; 1197 1198 if (params.needs_flush) { 1199 amdgpu_vm_tlb_flush(¶ms, fence, tlb_cb); 1200 tlb_cb = NULL; 1201 } 1202 1203 amdgpu_vm_pt_free_list(adev, ¶ms); 1204 1205 error_free: 1206 kfree(tlb_cb); 1207 amdgpu_vm_eviction_unlock(vm); 1208 drm_dev_exit(idx); 1209 return r; 1210 } 1211 1212 void amdgpu_vm_get_memory(struct amdgpu_vm *vm, 1213 struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM]) 1214 { 1215 spin_lock(&vm->status_lock); 1216 memcpy(stats, vm->stats, sizeof(*stats) * __AMDGPU_PL_NUM); 1217 spin_unlock(&vm->status_lock); 1218 } 1219 1220 /** 1221 * amdgpu_vm_bo_update - update all BO mappings in the vm page table 1222 * 1223 * @adev: amdgpu_device pointer 1224 * @bo_va: requested BO and VM object 1225 * @clear: if true clear the entries 1226 * 1227 * Fill in the page table entries for @bo_va. 1228 * 1229 * Returns: 1230 * 0 for success, -EINVAL for failure. 1231 */ 1232 int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va, 1233 bool clear) 1234 { 1235 struct amdgpu_bo *bo = bo_va->base.bo; 1236 struct amdgpu_vm *vm = bo_va->base.vm; 1237 struct amdgpu_bo_va_mapping *mapping; 1238 struct dma_fence **last_update; 1239 dma_addr_t *pages_addr = NULL; 1240 struct ttm_resource *mem; 1241 struct amdgpu_sync sync; 1242 bool flush_tlb = clear; 1243 uint64_t vram_base; 1244 uint64_t flags; 1245 bool uncached; 1246 int r; 1247 1248 amdgpu_sync_create(&sync); 1249 if (clear) { 1250 mem = NULL; 1251 1252 /* Implicitly sync to command submissions in the same VM before 1253 * unmapping. 1254 */ 1255 r = amdgpu_sync_resv(adev, &sync, vm->root.bo->tbo.base.resv, 1256 AMDGPU_SYNC_EQ_OWNER, vm); 1257 if (r) 1258 goto error_free; 1259 if (bo) { 1260 r = amdgpu_sync_kfd(&sync, bo->tbo.base.resv); 1261 if (r) 1262 goto error_free; 1263 } 1264 } else if (!bo) { 1265 mem = NULL; 1266 1267 /* PRT map operations don't need to sync to anything. */ 1268 1269 } else { 1270 struct drm_gem_object *obj = &bo->tbo.base; 1271 1272 if (obj->import_attach && bo_va->is_xgmi) { 1273 struct dma_buf *dma_buf = obj->import_attach->dmabuf; 1274 struct drm_gem_object *gobj = dma_buf->priv; 1275 struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj); 1276 1277 if (abo->tbo.resource && 1278 abo->tbo.resource->mem_type == TTM_PL_VRAM) 1279 bo = gem_to_amdgpu_bo(gobj); 1280 } 1281 mem = bo->tbo.resource; 1282 if (mem && (mem->mem_type == TTM_PL_TT || 1283 mem->mem_type == AMDGPU_PL_PREEMPT)) 1284 pages_addr = bo->tbo.ttm->dma_address; 1285 1286 /* Implicitly sync to moving fences before mapping anything */ 1287 r = amdgpu_sync_resv(adev, &sync, bo->tbo.base.resv, 1288 AMDGPU_SYNC_EXPLICIT, vm); 1289 if (r) 1290 goto error_free; 1291 } 1292 1293 if (bo) { 1294 struct amdgpu_device *bo_adev; 1295 1296 flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem); 1297 1298 if (amdgpu_bo_encrypted(bo)) 1299 flags |= AMDGPU_PTE_TMZ; 1300 1301 bo_adev = amdgpu_ttm_adev(bo->tbo.bdev); 1302 vram_base = bo_adev->vm_manager.vram_base_offset; 1303 uncached = (bo->flags & AMDGPU_GEM_CREATE_UNCACHED) != 0; 1304 } else { 1305 flags = 0x0; 1306 vram_base = 0; 1307 uncached = false; 1308 } 1309 1310 if (clear || amdgpu_vm_is_bo_always_valid(vm, bo)) 1311 last_update = &vm->last_update; 1312 else 1313 last_update = &bo_va->last_pt_update; 1314 1315 if (!clear && bo_va->base.moved) { 1316 flush_tlb = true; 1317 list_splice_init(&bo_va->valids, &bo_va->invalids); 1318 1319 } else if (bo_va->cleared != clear) { 1320 list_splice_init(&bo_va->valids, &bo_va->invalids); 1321 } 1322 1323 list_for_each_entry(mapping, &bo_va->invalids, list) { 1324 uint64_t update_flags = flags; 1325 1326 /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here 1327 * but in case of something, we filter the flags in first place 1328 */ 1329 if (!(mapping->flags & AMDGPU_PTE_READABLE)) 1330 update_flags &= ~AMDGPU_PTE_READABLE; 1331 if (!(mapping->flags & AMDGPU_PTE_WRITEABLE)) 1332 update_flags &= ~AMDGPU_PTE_WRITEABLE; 1333 1334 /* Apply ASIC specific mapping flags */ 1335 amdgpu_gmc_get_vm_pte(adev, mapping, &update_flags); 1336 1337 trace_amdgpu_vm_bo_update(mapping); 1338 1339 r = amdgpu_vm_update_range(adev, vm, false, false, flush_tlb, 1340 !uncached, &sync, mapping->start, 1341 mapping->last, update_flags, 1342 mapping->offset, vram_base, mem, 1343 pages_addr, last_update); 1344 if (r) 1345 goto error_free; 1346 } 1347 1348 /* If the BO is not in its preferred location add it back to 1349 * the evicted list so that it gets validated again on the 1350 * next command submission. 1351 */ 1352 if (amdgpu_vm_is_bo_always_valid(vm, bo)) { 1353 if (bo->tbo.resource && 1354 !(bo->preferred_domains & 1355 amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type))) 1356 amdgpu_vm_bo_evicted(&bo_va->base); 1357 else 1358 amdgpu_vm_bo_idle(&bo_va->base); 1359 } else { 1360 amdgpu_vm_bo_done(&bo_va->base); 1361 } 1362 1363 list_splice_init(&bo_va->invalids, &bo_va->valids); 1364 bo_va->cleared = clear; 1365 bo_va->base.moved = false; 1366 1367 if (trace_amdgpu_vm_bo_mapping_enabled()) { 1368 list_for_each_entry(mapping, &bo_va->valids, list) 1369 trace_amdgpu_vm_bo_mapping(mapping); 1370 } 1371 1372 error_free: 1373 amdgpu_sync_free(&sync); 1374 return r; 1375 } 1376 1377 /** 1378 * amdgpu_vm_update_prt_state - update the global PRT state 1379 * 1380 * @adev: amdgpu_device pointer 1381 */ 1382 static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev) 1383 { 1384 unsigned long flags; 1385 bool enable; 1386 1387 spin_lock_irqsave(&adev->vm_manager.prt_lock, flags); 1388 enable = !!atomic_read(&adev->vm_manager.num_prt_users); 1389 adev->gmc.gmc_funcs->set_prt(adev, enable); 1390 spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags); 1391 } 1392 1393 /** 1394 * amdgpu_vm_prt_get - add a PRT user 1395 * 1396 * @adev: amdgpu_device pointer 1397 */ 1398 static void amdgpu_vm_prt_get(struct amdgpu_device *adev) 1399 { 1400 if (!adev->gmc.gmc_funcs->set_prt) 1401 return; 1402 1403 if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1) 1404 amdgpu_vm_update_prt_state(adev); 1405 } 1406 1407 /** 1408 * amdgpu_vm_prt_put - drop a PRT user 1409 * 1410 * @adev: amdgpu_device pointer 1411 */ 1412 static void amdgpu_vm_prt_put(struct amdgpu_device *adev) 1413 { 1414 if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0) 1415 amdgpu_vm_update_prt_state(adev); 1416 } 1417 1418 /** 1419 * amdgpu_vm_prt_cb - callback for updating the PRT status 1420 * 1421 * @fence: fence for the callback 1422 * @_cb: the callback function 1423 */ 1424 static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb) 1425 { 1426 struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb); 1427 1428 amdgpu_vm_prt_put(cb->adev); 1429 kfree(cb); 1430 } 1431 1432 /** 1433 * amdgpu_vm_add_prt_cb - add callback for updating the PRT status 1434 * 1435 * @adev: amdgpu_device pointer 1436 * @fence: fence for the callback 1437 */ 1438 static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev, 1439 struct dma_fence *fence) 1440 { 1441 struct amdgpu_prt_cb *cb; 1442 1443 if (!adev->gmc.gmc_funcs->set_prt) 1444 return; 1445 1446 cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL); 1447 if (!cb) { 1448 /* Last resort when we are OOM */ 1449 if (fence) 1450 dma_fence_wait(fence, false); 1451 1452 amdgpu_vm_prt_put(adev); 1453 } else { 1454 cb->adev = adev; 1455 if (!fence || dma_fence_add_callback(fence, &cb->cb, 1456 amdgpu_vm_prt_cb)) 1457 amdgpu_vm_prt_cb(fence, &cb->cb); 1458 } 1459 } 1460 1461 /** 1462 * amdgpu_vm_free_mapping - free a mapping 1463 * 1464 * @adev: amdgpu_device pointer 1465 * @vm: requested vm 1466 * @mapping: mapping to be freed 1467 * @fence: fence of the unmap operation 1468 * 1469 * Free a mapping and make sure we decrease the PRT usage count if applicable. 1470 */ 1471 static void amdgpu_vm_free_mapping(struct amdgpu_device *adev, 1472 struct amdgpu_vm *vm, 1473 struct amdgpu_bo_va_mapping *mapping, 1474 struct dma_fence *fence) 1475 { 1476 if (mapping->flags & AMDGPU_PTE_PRT_FLAG(adev)) 1477 amdgpu_vm_add_prt_cb(adev, fence); 1478 kfree(mapping); 1479 } 1480 1481 /** 1482 * amdgpu_vm_prt_fini - finish all prt mappings 1483 * 1484 * @adev: amdgpu_device pointer 1485 * @vm: requested vm 1486 * 1487 * Register a cleanup callback to disable PRT support after VM dies. 1488 */ 1489 static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) 1490 { 1491 struct dma_resv *resv = vm->root.bo->tbo.base.resv; 1492 struct dma_resv_iter cursor; 1493 struct dma_fence *fence; 1494 1495 dma_resv_for_each_fence(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP, fence) { 1496 /* Add a callback for each fence in the reservation object */ 1497 amdgpu_vm_prt_get(adev); 1498 amdgpu_vm_add_prt_cb(adev, fence); 1499 } 1500 } 1501 1502 /** 1503 * amdgpu_vm_clear_freed - clear freed BOs in the PT 1504 * 1505 * @adev: amdgpu_device pointer 1506 * @vm: requested vm 1507 * @fence: optional resulting fence (unchanged if no work needed to be done 1508 * or if an error occurred) 1509 * 1510 * Make sure all freed BOs are cleared in the PT. 1511 * PTs have to be reserved and mutex must be locked! 1512 * 1513 * Returns: 1514 * 0 for success. 1515 * 1516 */ 1517 int amdgpu_vm_clear_freed(struct amdgpu_device *adev, 1518 struct amdgpu_vm *vm, 1519 struct dma_fence **fence) 1520 { 1521 struct amdgpu_bo_va_mapping *mapping; 1522 struct dma_fence *f = NULL; 1523 struct amdgpu_sync sync; 1524 int r; 1525 1526 1527 /* 1528 * Implicitly sync to command submissions in the same VM before 1529 * unmapping. 1530 */ 1531 amdgpu_sync_create(&sync); 1532 r = amdgpu_sync_resv(adev, &sync, vm->root.bo->tbo.base.resv, 1533 AMDGPU_SYNC_EQ_OWNER, vm); 1534 if (r) 1535 goto error_free; 1536 1537 while (!list_empty(&vm->freed)) { 1538 mapping = list_first_entry(&vm->freed, 1539 struct amdgpu_bo_va_mapping, list); 1540 list_del(&mapping->list); 1541 1542 r = amdgpu_vm_update_range(adev, vm, false, false, true, false, 1543 &sync, mapping->start, mapping->last, 1544 0, 0, 0, NULL, NULL, &f); 1545 amdgpu_vm_free_mapping(adev, vm, mapping, f); 1546 if (r) { 1547 dma_fence_put(f); 1548 goto error_free; 1549 } 1550 } 1551 1552 if (fence && f) { 1553 dma_fence_put(*fence); 1554 *fence = f; 1555 } else { 1556 dma_fence_put(f); 1557 } 1558 1559 error_free: 1560 amdgpu_sync_free(&sync); 1561 return r; 1562 1563 } 1564 1565 /** 1566 * amdgpu_vm_handle_moved - handle moved BOs in the PT 1567 * 1568 * @adev: amdgpu_device pointer 1569 * @vm: requested vm 1570 * @ticket: optional reservation ticket used to reserve the VM 1571 * 1572 * Make sure all BOs which are moved are updated in the PTs. 1573 * 1574 * Returns: 1575 * 0 for success. 1576 * 1577 * PTs have to be reserved! 1578 */ 1579 int amdgpu_vm_handle_moved(struct amdgpu_device *adev, 1580 struct amdgpu_vm *vm, 1581 struct ww_acquire_ctx *ticket) 1582 { 1583 struct amdgpu_bo_va *bo_va; 1584 struct dma_resv *resv; 1585 bool clear, unlock; 1586 int r; 1587 1588 spin_lock(&vm->status_lock); 1589 while (!list_empty(&vm->moved)) { 1590 bo_va = list_first_entry(&vm->moved, struct amdgpu_bo_va, 1591 base.vm_status); 1592 spin_unlock(&vm->status_lock); 1593 1594 /* Per VM BOs never need to bo cleared in the page tables */ 1595 r = amdgpu_vm_bo_update(adev, bo_va, false); 1596 if (r) 1597 return r; 1598 spin_lock(&vm->status_lock); 1599 } 1600 1601 while (!list_empty(&vm->invalidated)) { 1602 bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va, 1603 base.vm_status); 1604 resv = bo_va->base.bo->tbo.base.resv; 1605 spin_unlock(&vm->status_lock); 1606 1607 /* Try to reserve the BO to avoid clearing its ptes */ 1608 if (!adev->debug_vm && dma_resv_trylock(resv)) { 1609 clear = false; 1610 unlock = true; 1611 /* The caller is already holding the reservation lock */ 1612 } else if (ticket && dma_resv_locking_ctx(resv) == ticket) { 1613 clear = false; 1614 unlock = false; 1615 /* Somebody else is using the BO right now */ 1616 } else { 1617 clear = true; 1618 unlock = false; 1619 } 1620 1621 r = amdgpu_vm_bo_update(adev, bo_va, clear); 1622 1623 if (unlock) 1624 dma_resv_unlock(resv); 1625 if (r) 1626 return r; 1627 1628 /* Remember evicted DMABuf imports in compute VMs for later 1629 * validation 1630 */ 1631 if (vm->is_compute_context && 1632 bo_va->base.bo->tbo.base.import_attach && 1633 (!bo_va->base.bo->tbo.resource || 1634 bo_va->base.bo->tbo.resource->mem_type == TTM_PL_SYSTEM)) 1635 amdgpu_vm_bo_evicted_user(&bo_va->base); 1636 1637 spin_lock(&vm->status_lock); 1638 } 1639 spin_unlock(&vm->status_lock); 1640 1641 return 0; 1642 } 1643 1644 /** 1645 * amdgpu_vm_flush_compute_tlb - Flush TLB on compute VM 1646 * 1647 * @adev: amdgpu_device pointer 1648 * @vm: requested vm 1649 * @flush_type: flush type 1650 * @xcc_mask: mask of XCCs that belong to the compute partition in need of a TLB flush. 1651 * 1652 * Flush TLB if needed for a compute VM. 1653 * 1654 * Returns: 1655 * 0 for success. 1656 */ 1657 int amdgpu_vm_flush_compute_tlb(struct amdgpu_device *adev, 1658 struct amdgpu_vm *vm, 1659 uint32_t flush_type, 1660 uint32_t xcc_mask) 1661 { 1662 uint64_t tlb_seq = amdgpu_vm_tlb_seq(vm); 1663 bool all_hub = false; 1664 int xcc = 0, r = 0; 1665 1666 WARN_ON_ONCE(!vm->is_compute_context); 1667 1668 /* 1669 * It can be that we race and lose here, but that is extremely unlikely 1670 * and the worst thing which could happen is that we flush the changes 1671 * into the TLB once more which is harmless. 1672 */ 1673 if (atomic64_xchg(&vm->kfd_last_flushed_seq, tlb_seq) == tlb_seq) 1674 return 0; 1675 1676 if (adev->family == AMDGPU_FAMILY_AI || 1677 adev->family == AMDGPU_FAMILY_RV) 1678 all_hub = true; 1679 1680 for_each_inst(xcc, xcc_mask) { 1681 r = amdgpu_gmc_flush_gpu_tlb_pasid(adev, vm->pasid, flush_type, 1682 all_hub, xcc); 1683 if (r) 1684 break; 1685 } 1686 return r; 1687 } 1688 1689 /** 1690 * amdgpu_vm_bo_add - add a bo to a specific vm 1691 * 1692 * @adev: amdgpu_device pointer 1693 * @vm: requested vm 1694 * @bo: amdgpu buffer object 1695 * 1696 * Add @bo into the requested vm. 1697 * Add @bo to the list of bos associated with the vm 1698 * 1699 * Returns: 1700 * Newly added bo_va or NULL for failure 1701 * 1702 * Object has to be reserved! 1703 */ 1704 struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev, 1705 struct amdgpu_vm *vm, 1706 struct amdgpu_bo *bo) 1707 { 1708 struct amdgpu_bo_va *bo_va; 1709 1710 bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL); 1711 if (bo_va == NULL) { 1712 return NULL; 1713 } 1714 amdgpu_vm_bo_base_init(&bo_va->base, vm, bo); 1715 1716 bo_va->ref_count = 1; 1717 bo_va->last_pt_update = dma_fence_get_stub(); 1718 INIT_LIST_HEAD(&bo_va->valids); 1719 INIT_LIST_HEAD(&bo_va->invalids); 1720 1721 if (!bo) 1722 return bo_va; 1723 1724 dma_resv_assert_held(bo->tbo.base.resv); 1725 if (amdgpu_dmabuf_is_xgmi_accessible(adev, bo)) { 1726 bo_va->is_xgmi = true; 1727 /* Power up XGMI if it can be potentially used */ 1728 amdgpu_xgmi_set_pstate(adev, AMDGPU_XGMI_PSTATE_MAX_VEGA20); 1729 } 1730 1731 return bo_va; 1732 } 1733 1734 1735 /** 1736 * amdgpu_vm_bo_insert_map - insert a new mapping 1737 * 1738 * @adev: amdgpu_device pointer 1739 * @bo_va: bo_va to store the address 1740 * @mapping: the mapping to insert 1741 * 1742 * Insert a new mapping into all structures. 1743 */ 1744 static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev, 1745 struct amdgpu_bo_va *bo_va, 1746 struct amdgpu_bo_va_mapping *mapping) 1747 { 1748 struct amdgpu_vm *vm = bo_va->base.vm; 1749 struct amdgpu_bo *bo = bo_va->base.bo; 1750 1751 mapping->bo_va = bo_va; 1752 list_add(&mapping->list, &bo_va->invalids); 1753 amdgpu_vm_it_insert(mapping, &vm->va); 1754 1755 if (mapping->flags & AMDGPU_PTE_PRT_FLAG(adev)) 1756 amdgpu_vm_prt_get(adev); 1757 1758 if (amdgpu_vm_is_bo_always_valid(vm, bo) && !bo_va->base.moved) 1759 amdgpu_vm_bo_moved(&bo_va->base); 1760 1761 trace_amdgpu_vm_bo_map(bo_va, mapping); 1762 } 1763 1764 /* Validate operation parameters to prevent potential abuse */ 1765 static int amdgpu_vm_verify_parameters(struct amdgpu_device *adev, 1766 struct amdgpu_bo *bo, 1767 uint64_t saddr, 1768 uint64_t offset, 1769 uint64_t size) 1770 { 1771 uint64_t tmp, lpfn; 1772 1773 if (saddr & AMDGPU_GPU_PAGE_MASK 1774 || offset & AMDGPU_GPU_PAGE_MASK 1775 || size & AMDGPU_GPU_PAGE_MASK) 1776 return -EINVAL; 1777 1778 if (check_add_overflow(saddr, size, &tmp) 1779 || check_add_overflow(offset, size, &tmp) 1780 || size == 0 /* which also leads to end < begin */) 1781 return -EINVAL; 1782 1783 /* make sure object fit at this offset */ 1784 if (bo && offset + size > amdgpu_bo_size(bo)) 1785 return -EINVAL; 1786 1787 /* Ensure last pfn not exceed max_pfn */ 1788 lpfn = (saddr + size - 1) >> AMDGPU_GPU_PAGE_SHIFT; 1789 if (lpfn >= adev->vm_manager.max_pfn) 1790 return -EINVAL; 1791 1792 return 0; 1793 } 1794 1795 /** 1796 * amdgpu_vm_bo_map - map bo inside a vm 1797 * 1798 * @adev: amdgpu_device pointer 1799 * @bo_va: bo_va to store the address 1800 * @saddr: where to map the BO 1801 * @offset: requested offset in the BO 1802 * @size: BO size in bytes 1803 * @flags: attributes of pages (read/write/valid/etc.) 1804 * 1805 * Add a mapping of the BO at the specefied addr into the VM. 1806 * 1807 * Returns: 1808 * 0 for success, error for failure. 1809 * 1810 * Object has to be reserved and unreserved outside! 1811 */ 1812 int amdgpu_vm_bo_map(struct amdgpu_device *adev, 1813 struct amdgpu_bo_va *bo_va, 1814 uint64_t saddr, uint64_t offset, 1815 uint64_t size, uint64_t flags) 1816 { 1817 struct amdgpu_bo_va_mapping *mapping, *tmp; 1818 struct amdgpu_bo *bo = bo_va->base.bo; 1819 struct amdgpu_vm *vm = bo_va->base.vm; 1820 uint64_t eaddr; 1821 int r; 1822 1823 r = amdgpu_vm_verify_parameters(adev, bo, saddr, offset, size); 1824 if (r) 1825 return r; 1826 1827 saddr /= AMDGPU_GPU_PAGE_SIZE; 1828 eaddr = saddr + (size - 1) / AMDGPU_GPU_PAGE_SIZE; 1829 1830 tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr); 1831 if (tmp) { 1832 /* bo and tmp overlap, invalid addr */ 1833 dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with " 1834 "0x%010Lx-0x%010Lx\n", bo, saddr, eaddr, 1835 tmp->start, tmp->last + 1); 1836 return -EINVAL; 1837 } 1838 1839 mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); 1840 if (!mapping) 1841 return -ENOMEM; 1842 1843 mapping->start = saddr; 1844 mapping->last = eaddr; 1845 mapping->offset = offset; 1846 mapping->flags = flags; 1847 1848 amdgpu_vm_bo_insert_map(adev, bo_va, mapping); 1849 1850 return 0; 1851 } 1852 1853 /** 1854 * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings 1855 * 1856 * @adev: amdgpu_device pointer 1857 * @bo_va: bo_va to store the address 1858 * @saddr: where to map the BO 1859 * @offset: requested offset in the BO 1860 * @size: BO size in bytes 1861 * @flags: attributes of pages (read/write/valid/etc.) 1862 * 1863 * Add a mapping of the BO at the specefied addr into the VM. Replace existing 1864 * mappings as we do so. 1865 * 1866 * Returns: 1867 * 0 for success, error for failure. 1868 * 1869 * Object has to be reserved and unreserved outside! 1870 */ 1871 int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev, 1872 struct amdgpu_bo_va *bo_va, 1873 uint64_t saddr, uint64_t offset, 1874 uint64_t size, uint64_t flags) 1875 { 1876 struct amdgpu_bo_va_mapping *mapping; 1877 struct amdgpu_bo *bo = bo_va->base.bo; 1878 uint64_t eaddr; 1879 int r; 1880 1881 r = amdgpu_vm_verify_parameters(adev, bo, saddr, offset, size); 1882 if (r) 1883 return r; 1884 1885 /* Allocate all the needed memory */ 1886 mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); 1887 if (!mapping) 1888 return -ENOMEM; 1889 1890 r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size); 1891 if (r) { 1892 kfree(mapping); 1893 return r; 1894 } 1895 1896 saddr /= AMDGPU_GPU_PAGE_SIZE; 1897 eaddr = saddr + (size - 1) / AMDGPU_GPU_PAGE_SIZE; 1898 1899 mapping->start = saddr; 1900 mapping->last = eaddr; 1901 mapping->offset = offset; 1902 mapping->flags = flags; 1903 1904 amdgpu_vm_bo_insert_map(adev, bo_va, mapping); 1905 1906 return 0; 1907 } 1908 1909 /** 1910 * amdgpu_vm_bo_unmap - remove bo mapping from vm 1911 * 1912 * @adev: amdgpu_device pointer 1913 * @bo_va: bo_va to remove the address from 1914 * @saddr: where to the BO is mapped 1915 * 1916 * Remove a mapping of the BO at the specefied addr from the VM. 1917 * 1918 * Returns: 1919 * 0 for success, error for failure. 1920 * 1921 * Object has to be reserved and unreserved outside! 1922 */ 1923 int amdgpu_vm_bo_unmap(struct amdgpu_device *adev, 1924 struct amdgpu_bo_va *bo_va, 1925 uint64_t saddr) 1926 { 1927 struct amdgpu_bo_va_mapping *mapping; 1928 struct amdgpu_vm *vm = bo_va->base.vm; 1929 bool valid = true; 1930 1931 saddr /= AMDGPU_GPU_PAGE_SIZE; 1932 1933 list_for_each_entry(mapping, &bo_va->valids, list) { 1934 if (mapping->start == saddr) 1935 break; 1936 } 1937 1938 if (&mapping->list == &bo_va->valids) { 1939 valid = false; 1940 1941 list_for_each_entry(mapping, &bo_va->invalids, list) { 1942 if (mapping->start == saddr) 1943 break; 1944 } 1945 1946 if (&mapping->list == &bo_va->invalids) 1947 return -ENOENT; 1948 } 1949 1950 list_del(&mapping->list); 1951 amdgpu_vm_it_remove(mapping, &vm->va); 1952 mapping->bo_va = NULL; 1953 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 1954 1955 if (valid) 1956 list_add(&mapping->list, &vm->freed); 1957 else 1958 amdgpu_vm_free_mapping(adev, vm, mapping, 1959 bo_va->last_pt_update); 1960 1961 return 0; 1962 } 1963 1964 /** 1965 * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range 1966 * 1967 * @adev: amdgpu_device pointer 1968 * @vm: VM structure to use 1969 * @saddr: start of the range 1970 * @size: size of the range 1971 * 1972 * Remove all mappings in a range, split them as appropriate. 1973 * 1974 * Returns: 1975 * 0 for success, error for failure. 1976 */ 1977 int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev, 1978 struct amdgpu_vm *vm, 1979 uint64_t saddr, uint64_t size) 1980 { 1981 struct amdgpu_bo_va_mapping *before, *after, *tmp, *next; 1982 LIST_HEAD(removed); 1983 uint64_t eaddr; 1984 int r; 1985 1986 r = amdgpu_vm_verify_parameters(adev, NULL, saddr, 0, size); 1987 if (r) 1988 return r; 1989 1990 saddr /= AMDGPU_GPU_PAGE_SIZE; 1991 eaddr = saddr + (size - 1) / AMDGPU_GPU_PAGE_SIZE; 1992 1993 /* Allocate all the needed memory */ 1994 before = kzalloc(sizeof(*before), GFP_KERNEL); 1995 if (!before) 1996 return -ENOMEM; 1997 INIT_LIST_HEAD(&before->list); 1998 1999 after = kzalloc(sizeof(*after), GFP_KERNEL); 2000 if (!after) { 2001 kfree(before); 2002 return -ENOMEM; 2003 } 2004 INIT_LIST_HEAD(&after->list); 2005 2006 /* Now gather all removed mappings */ 2007 tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr); 2008 while (tmp) { 2009 /* Remember mapping split at the start */ 2010 if (tmp->start < saddr) { 2011 before->start = tmp->start; 2012 before->last = saddr - 1; 2013 before->offset = tmp->offset; 2014 before->flags = tmp->flags; 2015 before->bo_va = tmp->bo_va; 2016 list_add(&before->list, &tmp->bo_va->invalids); 2017 } 2018 2019 /* Remember mapping split at the end */ 2020 if (tmp->last > eaddr) { 2021 after->start = eaddr + 1; 2022 after->last = tmp->last; 2023 after->offset = tmp->offset; 2024 after->offset += (after->start - tmp->start) << PAGE_SHIFT; 2025 after->flags = tmp->flags; 2026 after->bo_va = tmp->bo_va; 2027 list_add(&after->list, &tmp->bo_va->invalids); 2028 } 2029 2030 list_del(&tmp->list); 2031 list_add(&tmp->list, &removed); 2032 2033 tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr); 2034 } 2035 2036 /* And free them up */ 2037 list_for_each_entry_safe(tmp, next, &removed, list) { 2038 amdgpu_vm_it_remove(tmp, &vm->va); 2039 list_del(&tmp->list); 2040 2041 if (tmp->start < saddr) 2042 tmp->start = saddr; 2043 if (tmp->last > eaddr) 2044 tmp->last = eaddr; 2045 2046 tmp->bo_va = NULL; 2047 list_add(&tmp->list, &vm->freed); 2048 trace_amdgpu_vm_bo_unmap(NULL, tmp); 2049 } 2050 2051 /* Insert partial mapping before the range */ 2052 if (!list_empty(&before->list)) { 2053 struct amdgpu_bo *bo = before->bo_va->base.bo; 2054 2055 amdgpu_vm_it_insert(before, &vm->va); 2056 if (before->flags & AMDGPU_PTE_PRT_FLAG(adev)) 2057 amdgpu_vm_prt_get(adev); 2058 2059 if (amdgpu_vm_is_bo_always_valid(vm, bo) && 2060 !before->bo_va->base.moved) 2061 amdgpu_vm_bo_moved(&before->bo_va->base); 2062 } else { 2063 kfree(before); 2064 } 2065 2066 /* Insert partial mapping after the range */ 2067 if (!list_empty(&after->list)) { 2068 struct amdgpu_bo *bo = after->bo_va->base.bo; 2069 2070 amdgpu_vm_it_insert(after, &vm->va); 2071 if (after->flags & AMDGPU_PTE_PRT_FLAG(adev)) 2072 amdgpu_vm_prt_get(adev); 2073 2074 if (amdgpu_vm_is_bo_always_valid(vm, bo) && 2075 !after->bo_va->base.moved) 2076 amdgpu_vm_bo_moved(&after->bo_va->base); 2077 } else { 2078 kfree(after); 2079 } 2080 2081 return 0; 2082 } 2083 2084 /** 2085 * amdgpu_vm_bo_lookup_mapping - find mapping by address 2086 * 2087 * @vm: the requested VM 2088 * @addr: the address 2089 * 2090 * Find a mapping by it's address. 2091 * 2092 * Returns: 2093 * The amdgpu_bo_va_mapping matching for addr or NULL 2094 * 2095 */ 2096 struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm, 2097 uint64_t addr) 2098 { 2099 return amdgpu_vm_it_iter_first(&vm->va, addr, addr); 2100 } 2101 2102 /** 2103 * amdgpu_vm_bo_trace_cs - trace all reserved mappings 2104 * 2105 * @vm: the requested vm 2106 * @ticket: CS ticket 2107 * 2108 * Trace all mappings of BOs reserved during a command submission. 2109 */ 2110 void amdgpu_vm_bo_trace_cs(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket) 2111 { 2112 struct amdgpu_bo_va_mapping *mapping; 2113 2114 if (!trace_amdgpu_vm_bo_cs_enabled()) 2115 return; 2116 2117 for (mapping = amdgpu_vm_it_iter_first(&vm->va, 0, U64_MAX); mapping; 2118 mapping = amdgpu_vm_it_iter_next(mapping, 0, U64_MAX)) { 2119 if (mapping->bo_va && mapping->bo_va->base.bo) { 2120 struct amdgpu_bo *bo; 2121 2122 bo = mapping->bo_va->base.bo; 2123 if (dma_resv_locking_ctx(bo->tbo.base.resv) != 2124 ticket) 2125 continue; 2126 } 2127 2128 trace_amdgpu_vm_bo_cs(mapping); 2129 } 2130 } 2131 2132 /** 2133 * amdgpu_vm_bo_del - remove a bo from a specific vm 2134 * 2135 * @adev: amdgpu_device pointer 2136 * @bo_va: requested bo_va 2137 * 2138 * Remove @bo_va->bo from the requested vm. 2139 * 2140 * Object have to be reserved! 2141 */ 2142 void amdgpu_vm_bo_del(struct amdgpu_device *adev, 2143 struct amdgpu_bo_va *bo_va) 2144 { 2145 struct amdgpu_bo_va_mapping *mapping, *next; 2146 struct amdgpu_bo *bo = bo_va->base.bo; 2147 struct amdgpu_vm *vm = bo_va->base.vm; 2148 struct amdgpu_vm_bo_base **base; 2149 2150 dma_resv_assert_held(vm->root.bo->tbo.base.resv); 2151 2152 if (bo) { 2153 dma_resv_assert_held(bo->tbo.base.resv); 2154 if (amdgpu_vm_is_bo_always_valid(vm, bo)) 2155 ttm_bo_set_bulk_move(&bo->tbo, NULL); 2156 2157 for (base = &bo_va->base.bo->vm_bo; *base; 2158 base = &(*base)->next) { 2159 if (*base != &bo_va->base) 2160 continue; 2161 2162 amdgpu_vm_update_stats(*base, bo->tbo.resource, -1); 2163 *base = bo_va->base.next; 2164 break; 2165 } 2166 } 2167 2168 spin_lock(&vm->status_lock); 2169 list_del(&bo_va->base.vm_status); 2170 spin_unlock(&vm->status_lock); 2171 2172 list_for_each_entry_safe(mapping, next, &bo_va->valids, list) { 2173 list_del(&mapping->list); 2174 amdgpu_vm_it_remove(mapping, &vm->va); 2175 mapping->bo_va = NULL; 2176 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 2177 list_add(&mapping->list, &vm->freed); 2178 } 2179 list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) { 2180 list_del(&mapping->list); 2181 amdgpu_vm_it_remove(mapping, &vm->va); 2182 amdgpu_vm_free_mapping(adev, vm, mapping, 2183 bo_va->last_pt_update); 2184 } 2185 2186 dma_fence_put(bo_va->last_pt_update); 2187 2188 if (bo && bo_va->is_xgmi) 2189 amdgpu_xgmi_set_pstate(adev, AMDGPU_XGMI_PSTATE_MIN); 2190 2191 kfree(bo_va); 2192 } 2193 2194 /** 2195 * amdgpu_vm_evictable - check if we can evict a VM 2196 * 2197 * @bo: A page table of the VM. 2198 * 2199 * Check if it is possible to evict a VM. 2200 */ 2201 bool amdgpu_vm_evictable(struct amdgpu_bo *bo) 2202 { 2203 struct amdgpu_vm_bo_base *bo_base = bo->vm_bo; 2204 2205 /* Page tables of a destroyed VM can go away immediately */ 2206 if (!bo_base || !bo_base->vm) 2207 return true; 2208 2209 /* Don't evict VM page tables while they are busy */ 2210 if (!dma_resv_test_signaled(bo->tbo.base.resv, DMA_RESV_USAGE_BOOKKEEP)) 2211 return false; 2212 2213 /* Try to block ongoing updates */ 2214 if (!amdgpu_vm_eviction_trylock(bo_base->vm)) 2215 return false; 2216 2217 /* Don't evict VM page tables while they are updated */ 2218 if (!dma_fence_is_signaled(bo_base->vm->last_unlocked)) { 2219 amdgpu_vm_eviction_unlock(bo_base->vm); 2220 return false; 2221 } 2222 2223 bo_base->vm->evicting = true; 2224 amdgpu_vm_eviction_unlock(bo_base->vm); 2225 return true; 2226 } 2227 2228 /** 2229 * amdgpu_vm_bo_invalidate - mark the bo as invalid 2230 * 2231 * @bo: amdgpu buffer object 2232 * @evicted: is the BO evicted 2233 * 2234 * Mark @bo as invalid. 2235 */ 2236 void amdgpu_vm_bo_invalidate(struct amdgpu_bo *bo, bool evicted) 2237 { 2238 struct amdgpu_vm_bo_base *bo_base; 2239 2240 for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) { 2241 struct amdgpu_vm *vm = bo_base->vm; 2242 2243 if (evicted && amdgpu_vm_is_bo_always_valid(vm, bo)) { 2244 amdgpu_vm_bo_evicted(bo_base); 2245 continue; 2246 } 2247 2248 if (bo_base->moved) 2249 continue; 2250 bo_base->moved = true; 2251 2252 if (bo->tbo.type == ttm_bo_type_kernel) 2253 amdgpu_vm_bo_relocated(bo_base); 2254 else if (amdgpu_vm_is_bo_always_valid(vm, bo)) 2255 amdgpu_vm_bo_moved(bo_base); 2256 else 2257 amdgpu_vm_bo_invalidated(bo_base); 2258 } 2259 } 2260 2261 /** 2262 * amdgpu_vm_bo_move - handle BO move 2263 * 2264 * @bo: amdgpu buffer object 2265 * @new_mem: the new placement of the BO move 2266 * @evicted: is the BO evicted 2267 * 2268 * Update the memory stats for the new placement and mark @bo as invalid. 2269 */ 2270 void amdgpu_vm_bo_move(struct amdgpu_bo *bo, struct ttm_resource *new_mem, 2271 bool evicted) 2272 { 2273 struct amdgpu_vm_bo_base *bo_base; 2274 2275 for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) { 2276 struct amdgpu_vm *vm = bo_base->vm; 2277 2278 spin_lock(&vm->status_lock); 2279 amdgpu_vm_update_stats_locked(bo_base, bo->tbo.resource, -1); 2280 amdgpu_vm_update_stats_locked(bo_base, new_mem, +1); 2281 spin_unlock(&vm->status_lock); 2282 } 2283 2284 amdgpu_vm_bo_invalidate(bo, evicted); 2285 } 2286 2287 /** 2288 * amdgpu_vm_get_block_size - calculate VM page table size as power of two 2289 * 2290 * @vm_size: VM size 2291 * 2292 * Returns: 2293 * VM page table as power of two 2294 */ 2295 static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size) 2296 { 2297 /* Total bits covered by PD + PTs */ 2298 unsigned bits = ilog2(vm_size) + 18; 2299 2300 /* Make sure the PD is 4K in size up to 8GB address space. 2301 Above that split equal between PD and PTs */ 2302 if (vm_size <= 8) 2303 return (bits - 9); 2304 else 2305 return ((bits + 3) / 2); 2306 } 2307 2308 /** 2309 * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size 2310 * 2311 * @adev: amdgpu_device pointer 2312 * @min_vm_size: the minimum vm size in GB if it's set auto 2313 * @fragment_size_default: Default PTE fragment size 2314 * @max_level: max VMPT level 2315 * @max_bits: max address space size in bits 2316 * 2317 */ 2318 void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size, 2319 uint32_t fragment_size_default, unsigned max_level, 2320 unsigned max_bits) 2321 { 2322 unsigned int max_size = 1 << (max_bits - 30); 2323 unsigned int vm_size; 2324 uint64_t tmp; 2325 2326 /* adjust vm size first */ 2327 if (amdgpu_vm_size != -1) { 2328 vm_size = amdgpu_vm_size; 2329 if (vm_size > max_size) { 2330 dev_warn(adev->dev, "VM size (%d) too large, max is %u GB\n", 2331 amdgpu_vm_size, max_size); 2332 vm_size = max_size; 2333 } 2334 } else { 2335 struct sysinfo si; 2336 unsigned int phys_ram_gb; 2337 2338 /* Optimal VM size depends on the amount of physical 2339 * RAM available. Underlying requirements and 2340 * assumptions: 2341 * 2342 * - Need to map system memory and VRAM from all GPUs 2343 * - VRAM from other GPUs not known here 2344 * - Assume VRAM <= system memory 2345 * - On GFX8 and older, VM space can be segmented for 2346 * different MTYPEs 2347 * - Need to allow room for fragmentation, guard pages etc. 2348 * 2349 * This adds up to a rough guess of system memory x3. 2350 * Round up to power of two to maximize the available 2351 * VM size with the given page table size. 2352 */ 2353 si_meminfo(&si); 2354 phys_ram_gb = ((uint64_t)si.totalram * si.mem_unit + 2355 (1 << 30) - 1) >> 30; 2356 vm_size = roundup_pow_of_two( 2357 clamp(phys_ram_gb * 3, min_vm_size, max_size)); 2358 } 2359 2360 adev->vm_manager.max_pfn = (uint64_t)vm_size << 18; 2361 2362 tmp = roundup_pow_of_two(adev->vm_manager.max_pfn); 2363 if (amdgpu_vm_block_size != -1) 2364 tmp >>= amdgpu_vm_block_size - 9; 2365 tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1; 2366 adev->vm_manager.num_level = min_t(unsigned int, max_level, tmp); 2367 switch (adev->vm_manager.num_level) { 2368 case 3: 2369 adev->vm_manager.root_level = AMDGPU_VM_PDB2; 2370 break; 2371 case 2: 2372 adev->vm_manager.root_level = AMDGPU_VM_PDB1; 2373 break; 2374 case 1: 2375 adev->vm_manager.root_level = AMDGPU_VM_PDB0; 2376 break; 2377 default: 2378 dev_err(adev->dev, "VMPT only supports 2~4+1 levels\n"); 2379 } 2380 /* block size depends on vm size and hw setup*/ 2381 if (amdgpu_vm_block_size != -1) 2382 adev->vm_manager.block_size = 2383 min((unsigned)amdgpu_vm_block_size, max_bits 2384 - AMDGPU_GPU_PAGE_SHIFT 2385 - 9 * adev->vm_manager.num_level); 2386 else if (adev->vm_manager.num_level > 1) 2387 adev->vm_manager.block_size = 9; 2388 else 2389 adev->vm_manager.block_size = amdgpu_vm_get_block_size(tmp); 2390 2391 if (amdgpu_vm_fragment_size == -1) 2392 adev->vm_manager.fragment_size = fragment_size_default; 2393 else 2394 adev->vm_manager.fragment_size = amdgpu_vm_fragment_size; 2395 2396 DRM_INFO("vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n", 2397 vm_size, adev->vm_manager.num_level + 1, 2398 adev->vm_manager.block_size, 2399 adev->vm_manager.fragment_size); 2400 } 2401 2402 /** 2403 * amdgpu_vm_wait_idle - wait for the VM to become idle 2404 * 2405 * @vm: VM object to wait for 2406 * @timeout: timeout to wait for VM to become idle 2407 */ 2408 long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout) 2409 { 2410 timeout = dma_resv_wait_timeout(vm->root.bo->tbo.base.resv, 2411 DMA_RESV_USAGE_BOOKKEEP, 2412 true, timeout); 2413 if (timeout <= 0) 2414 return timeout; 2415 2416 return dma_fence_wait_timeout(vm->last_unlocked, true, timeout); 2417 } 2418 2419 static void amdgpu_vm_destroy_task_info(struct kref *kref) 2420 { 2421 struct amdgpu_task_info *ti = container_of(kref, struct amdgpu_task_info, refcount); 2422 2423 kfree(ti); 2424 } 2425 2426 static inline struct amdgpu_vm * 2427 amdgpu_vm_get_vm_from_pasid(struct amdgpu_device *adev, u32 pasid) 2428 { 2429 struct amdgpu_vm *vm; 2430 unsigned long flags; 2431 2432 xa_lock_irqsave(&adev->vm_manager.pasids, flags); 2433 vm = xa_load(&adev->vm_manager.pasids, pasid); 2434 xa_unlock_irqrestore(&adev->vm_manager.pasids, flags); 2435 2436 return vm; 2437 } 2438 2439 /** 2440 * amdgpu_vm_put_task_info - reference down the vm task_info ptr 2441 * 2442 * @task_info: task_info struct under discussion. 2443 * 2444 * frees the vm task_info ptr at the last put 2445 */ 2446 void amdgpu_vm_put_task_info(struct amdgpu_task_info *task_info) 2447 { 2448 kref_put(&task_info->refcount, amdgpu_vm_destroy_task_info); 2449 } 2450 2451 /** 2452 * amdgpu_vm_get_task_info_vm - Extracts task info for a vm. 2453 * 2454 * @vm: VM to get info from 2455 * 2456 * Returns the reference counted task_info structure, which must be 2457 * referenced down with amdgpu_vm_put_task_info. 2458 */ 2459 struct amdgpu_task_info * 2460 amdgpu_vm_get_task_info_vm(struct amdgpu_vm *vm) 2461 { 2462 struct amdgpu_task_info *ti = NULL; 2463 2464 if (vm) { 2465 ti = vm->task_info; 2466 kref_get(&vm->task_info->refcount); 2467 } 2468 2469 return ti; 2470 } 2471 2472 /** 2473 * amdgpu_vm_get_task_info_pasid - Extracts task info for a PASID. 2474 * 2475 * @adev: drm device pointer 2476 * @pasid: PASID identifier for VM 2477 * 2478 * Returns the reference counted task_info structure, which must be 2479 * referenced down with amdgpu_vm_put_task_info. 2480 */ 2481 struct amdgpu_task_info * 2482 amdgpu_vm_get_task_info_pasid(struct amdgpu_device *adev, u32 pasid) 2483 { 2484 return amdgpu_vm_get_task_info_vm( 2485 amdgpu_vm_get_vm_from_pasid(adev, pasid)); 2486 } 2487 2488 static int amdgpu_vm_create_task_info(struct amdgpu_vm *vm) 2489 { 2490 vm->task_info = kzalloc(sizeof(struct amdgpu_task_info), GFP_KERNEL); 2491 if (!vm->task_info) 2492 return -ENOMEM; 2493 2494 kref_init(&vm->task_info->refcount); 2495 return 0; 2496 } 2497 2498 /** 2499 * amdgpu_vm_set_task_info - Sets VMs task info. 2500 * 2501 * @vm: vm for which to set the info 2502 */ 2503 void amdgpu_vm_set_task_info(struct amdgpu_vm *vm) 2504 { 2505 if (!vm->task_info) 2506 return; 2507 2508 if (vm->task_info->pid == current->pid) 2509 return; 2510 2511 vm->task_info->pid = current->pid; 2512 get_task_comm(vm->task_info->task_name, current); 2513 2514 if (current->group_leader->mm != current->mm) 2515 return; 2516 2517 vm->task_info->tgid = current->group_leader->pid; 2518 get_task_comm(vm->task_info->process_name, current->group_leader); 2519 } 2520 2521 /** 2522 * amdgpu_vm_init - initialize a vm instance 2523 * 2524 * @adev: amdgpu_device pointer 2525 * @vm: requested vm 2526 * @xcp_id: GPU partition selection id 2527 * 2528 * Init @vm fields. 2529 * 2530 * Returns: 2531 * 0 for success, error for failure. 2532 */ 2533 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, 2534 int32_t xcp_id) 2535 { 2536 struct amdgpu_bo *root_bo; 2537 struct amdgpu_bo_vm *root; 2538 int r, i; 2539 2540 vm->va = RB_ROOT_CACHED; 2541 for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) 2542 vm->reserved_vmid[i] = NULL; 2543 INIT_LIST_HEAD(&vm->evicted); 2544 INIT_LIST_HEAD(&vm->evicted_user); 2545 INIT_LIST_HEAD(&vm->relocated); 2546 INIT_LIST_HEAD(&vm->moved); 2547 INIT_LIST_HEAD(&vm->idle); 2548 INIT_LIST_HEAD(&vm->invalidated); 2549 spin_lock_init(&vm->status_lock); 2550 INIT_LIST_HEAD(&vm->freed); 2551 INIT_LIST_HEAD(&vm->done); 2552 INIT_KFIFO(vm->faults); 2553 2554 r = amdgpu_vm_init_entities(adev, vm); 2555 if (r) 2556 return r; 2557 2558 ttm_lru_bulk_move_init(&vm->lru_bulk_move); 2559 2560 vm->is_compute_context = false; 2561 2562 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode & 2563 AMDGPU_VM_USE_CPU_FOR_GFX); 2564 2565 DRM_DEBUG_DRIVER("VM update mode is %s\n", 2566 vm->use_cpu_for_update ? "CPU" : "SDMA"); 2567 WARN_ONCE((vm->use_cpu_for_update && 2568 !amdgpu_gmc_vram_full_visible(&adev->gmc)), 2569 "CPU update of VM recommended only for large BAR system\n"); 2570 2571 if (vm->use_cpu_for_update) 2572 vm->update_funcs = &amdgpu_vm_cpu_funcs; 2573 else 2574 vm->update_funcs = &amdgpu_vm_sdma_funcs; 2575 2576 vm->last_update = dma_fence_get_stub(); 2577 vm->last_unlocked = dma_fence_get_stub(); 2578 vm->last_tlb_flush = dma_fence_get_stub(); 2579 vm->generation = amdgpu_vm_generation(adev, NULL); 2580 2581 mutex_init(&vm->eviction_lock); 2582 vm->evicting = false; 2583 vm->tlb_fence_context = dma_fence_context_alloc(1); 2584 2585 r = amdgpu_vm_pt_create(adev, vm, adev->vm_manager.root_level, 2586 false, &root, xcp_id); 2587 if (r) 2588 goto error_free_delayed; 2589 2590 root_bo = amdgpu_bo_ref(&root->bo); 2591 r = amdgpu_bo_reserve(root_bo, true); 2592 if (r) { 2593 amdgpu_bo_unref(&root_bo); 2594 goto error_free_delayed; 2595 } 2596 2597 amdgpu_vm_bo_base_init(&vm->root, vm, root_bo); 2598 r = dma_resv_reserve_fences(root_bo->tbo.base.resv, 1); 2599 if (r) 2600 goto error_free_root; 2601 2602 r = amdgpu_vm_pt_clear(adev, vm, root, false); 2603 if (r) 2604 goto error_free_root; 2605 2606 r = amdgpu_vm_create_task_info(vm); 2607 if (r) 2608 DRM_DEBUG("Failed to create task info for VM\n"); 2609 2610 amdgpu_bo_unreserve(vm->root.bo); 2611 amdgpu_bo_unref(&root_bo); 2612 2613 return 0; 2614 2615 error_free_root: 2616 amdgpu_vm_pt_free_root(adev, vm); 2617 amdgpu_bo_unreserve(vm->root.bo); 2618 amdgpu_bo_unref(&root_bo); 2619 2620 error_free_delayed: 2621 dma_fence_put(vm->last_tlb_flush); 2622 dma_fence_put(vm->last_unlocked); 2623 ttm_lru_bulk_move_fini(&adev->mman.bdev, &vm->lru_bulk_move); 2624 amdgpu_vm_fini_entities(vm); 2625 2626 return r; 2627 } 2628 2629 /** 2630 * amdgpu_vm_make_compute - Turn a GFX VM into a compute VM 2631 * 2632 * @adev: amdgpu_device pointer 2633 * @vm: requested vm 2634 * 2635 * This only works on GFX VMs that don't have any BOs added and no 2636 * page tables allocated yet. 2637 * 2638 * Changes the following VM parameters: 2639 * - use_cpu_for_update 2640 * - pte_supports_ats 2641 * 2642 * Reinitializes the page directory to reflect the changed ATS 2643 * setting. 2644 * 2645 * Returns: 2646 * 0 for success, -errno for errors. 2647 */ 2648 int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm) 2649 { 2650 int r; 2651 2652 r = amdgpu_bo_reserve(vm->root.bo, true); 2653 if (r) 2654 return r; 2655 2656 /* Update VM state */ 2657 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode & 2658 AMDGPU_VM_USE_CPU_FOR_COMPUTE); 2659 DRM_DEBUG_DRIVER("VM update mode is %s\n", 2660 vm->use_cpu_for_update ? "CPU" : "SDMA"); 2661 WARN_ONCE((vm->use_cpu_for_update && 2662 !amdgpu_gmc_vram_full_visible(&adev->gmc)), 2663 "CPU update of VM recommended only for large BAR system\n"); 2664 2665 if (vm->use_cpu_for_update) { 2666 /* Sync with last SDMA update/clear before switching to CPU */ 2667 r = amdgpu_bo_sync_wait(vm->root.bo, 2668 AMDGPU_FENCE_OWNER_UNDEFINED, true); 2669 if (r) 2670 goto unreserve_bo; 2671 2672 vm->update_funcs = &amdgpu_vm_cpu_funcs; 2673 r = amdgpu_vm_pt_map_tables(adev, vm); 2674 if (r) 2675 goto unreserve_bo; 2676 2677 } else { 2678 vm->update_funcs = &amdgpu_vm_sdma_funcs; 2679 } 2680 2681 dma_fence_put(vm->last_update); 2682 vm->last_update = dma_fence_get_stub(); 2683 vm->is_compute_context = true; 2684 2685 unreserve_bo: 2686 amdgpu_bo_unreserve(vm->root.bo); 2687 return r; 2688 } 2689 2690 static int amdgpu_vm_stats_is_zero(struct amdgpu_vm *vm) 2691 { 2692 for (int i = 0; i < __AMDGPU_PL_NUM; ++i) { 2693 if (!(drm_memory_stats_is_zero(&vm->stats[i].drm) && 2694 vm->stats[i].evicted == 0)) 2695 return false; 2696 } 2697 return true; 2698 } 2699 2700 /** 2701 * amdgpu_vm_fini - tear down a vm instance 2702 * 2703 * @adev: amdgpu_device pointer 2704 * @vm: requested vm 2705 * 2706 * Tear down @vm. 2707 * Unbind the VM and remove all bos from the vm bo list 2708 */ 2709 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) 2710 { 2711 struct amdgpu_bo_va_mapping *mapping, *tmp; 2712 bool prt_fini_needed = !!adev->gmc.gmc_funcs->set_prt; 2713 struct amdgpu_bo *root; 2714 unsigned long flags; 2715 int i; 2716 2717 amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm); 2718 2719 root = amdgpu_bo_ref(vm->root.bo); 2720 amdgpu_bo_reserve(root, true); 2721 amdgpu_vm_set_pasid(adev, vm, 0); 2722 dma_fence_wait(vm->last_unlocked, false); 2723 dma_fence_put(vm->last_unlocked); 2724 dma_fence_wait(vm->last_tlb_flush, false); 2725 /* Make sure that all fence callbacks have completed */ 2726 spin_lock_irqsave(vm->last_tlb_flush->lock, flags); 2727 spin_unlock_irqrestore(vm->last_tlb_flush->lock, flags); 2728 dma_fence_put(vm->last_tlb_flush); 2729 2730 list_for_each_entry_safe(mapping, tmp, &vm->freed, list) { 2731 if (mapping->flags & AMDGPU_PTE_PRT_FLAG(adev) && prt_fini_needed) { 2732 amdgpu_vm_prt_fini(adev, vm); 2733 prt_fini_needed = false; 2734 } 2735 2736 list_del(&mapping->list); 2737 amdgpu_vm_free_mapping(adev, vm, mapping, NULL); 2738 } 2739 2740 amdgpu_vm_pt_free_root(adev, vm); 2741 amdgpu_bo_unreserve(root); 2742 amdgpu_bo_unref(&root); 2743 WARN_ON(vm->root.bo); 2744 2745 amdgpu_vm_fini_entities(vm); 2746 2747 if (!RB_EMPTY_ROOT(&vm->va.rb_root)) { 2748 dev_err(adev->dev, "still active bo inside vm\n"); 2749 } 2750 rbtree_postorder_for_each_entry_safe(mapping, tmp, 2751 &vm->va.rb_root, rb) { 2752 /* Don't remove the mapping here, we don't want to trigger a 2753 * rebalance and the tree is about to be destroyed anyway. 2754 */ 2755 list_del(&mapping->list); 2756 kfree(mapping); 2757 } 2758 2759 dma_fence_put(vm->last_update); 2760 2761 for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) { 2762 if (vm->reserved_vmid[i]) { 2763 amdgpu_vmid_free_reserved(adev, i); 2764 vm->reserved_vmid[i] = false; 2765 } 2766 } 2767 2768 ttm_lru_bulk_move_fini(&adev->mman.bdev, &vm->lru_bulk_move); 2769 2770 if (!amdgpu_vm_stats_is_zero(vm)) { 2771 struct amdgpu_task_info *ti = vm->task_info; 2772 2773 dev_warn(adev->dev, 2774 "VM memory stats for proc %s(%d) task %s(%d) is non-zero when fini\n", 2775 ti->process_name, ti->pid, ti->task_name, ti->tgid); 2776 } 2777 2778 amdgpu_vm_put_task_info(vm->task_info); 2779 } 2780 2781 /** 2782 * amdgpu_vm_manager_init - init the VM manager 2783 * 2784 * @adev: amdgpu_device pointer 2785 * 2786 * Initialize the VM manager structures 2787 */ 2788 void amdgpu_vm_manager_init(struct amdgpu_device *adev) 2789 { 2790 unsigned i; 2791 2792 /* Concurrent flushes are only possible starting with Vega10 and 2793 * are broken on Navi10 and Navi14. 2794 */ 2795 adev->vm_manager.concurrent_flush = !(adev->asic_type < CHIP_VEGA10 || 2796 adev->asic_type == CHIP_NAVI10 || 2797 adev->asic_type == CHIP_NAVI14); 2798 amdgpu_vmid_mgr_init(adev); 2799 2800 adev->vm_manager.fence_context = 2801 dma_fence_context_alloc(AMDGPU_MAX_RINGS); 2802 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) 2803 adev->vm_manager.seqno[i] = 0; 2804 2805 spin_lock_init(&adev->vm_manager.prt_lock); 2806 atomic_set(&adev->vm_manager.num_prt_users, 0); 2807 2808 /* If not overridden by the user, by default, only in large BAR systems 2809 * Compute VM tables will be updated by CPU 2810 */ 2811 #ifdef CONFIG_X86_64 2812 if (amdgpu_vm_update_mode == -1) { 2813 /* For asic with VF MMIO access protection 2814 * avoid using CPU for VM table updates 2815 */ 2816 if (amdgpu_gmc_vram_full_visible(&adev->gmc) && 2817 !amdgpu_sriov_vf_mmio_access_protection(adev)) 2818 adev->vm_manager.vm_update_mode = 2819 AMDGPU_VM_USE_CPU_FOR_COMPUTE; 2820 else 2821 adev->vm_manager.vm_update_mode = 0; 2822 } else 2823 adev->vm_manager.vm_update_mode = amdgpu_vm_update_mode; 2824 #else 2825 adev->vm_manager.vm_update_mode = 0; 2826 #endif 2827 2828 xa_init_flags(&adev->vm_manager.pasids, XA_FLAGS_LOCK_IRQ); 2829 } 2830 2831 /** 2832 * amdgpu_vm_manager_fini - cleanup VM manager 2833 * 2834 * @adev: amdgpu_device pointer 2835 * 2836 * Cleanup the VM manager and free resources. 2837 */ 2838 void amdgpu_vm_manager_fini(struct amdgpu_device *adev) 2839 { 2840 WARN_ON(!xa_empty(&adev->vm_manager.pasids)); 2841 xa_destroy(&adev->vm_manager.pasids); 2842 2843 amdgpu_vmid_mgr_fini(adev); 2844 } 2845 2846 /** 2847 * amdgpu_vm_ioctl - Manages VMID reservation for vm hubs. 2848 * 2849 * @dev: drm device pointer 2850 * @data: drm_amdgpu_vm 2851 * @filp: drm file pointer 2852 * 2853 * Returns: 2854 * 0 for success, -errno for errors. 2855 */ 2856 int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 2857 { 2858 union drm_amdgpu_vm *args = data; 2859 struct amdgpu_device *adev = drm_to_adev(dev); 2860 struct amdgpu_fpriv *fpriv = filp->driver_priv; 2861 2862 /* No valid flags defined yet */ 2863 if (args->in.flags) 2864 return -EINVAL; 2865 2866 switch (args->in.op) { 2867 case AMDGPU_VM_OP_RESERVE_VMID: 2868 /* We only have requirement to reserve vmid from gfxhub */ 2869 if (!fpriv->vm.reserved_vmid[AMDGPU_GFXHUB(0)]) { 2870 amdgpu_vmid_alloc_reserved(adev, AMDGPU_GFXHUB(0)); 2871 fpriv->vm.reserved_vmid[AMDGPU_GFXHUB(0)] = true; 2872 } 2873 2874 break; 2875 case AMDGPU_VM_OP_UNRESERVE_VMID: 2876 if (fpriv->vm.reserved_vmid[AMDGPU_GFXHUB(0)]) { 2877 amdgpu_vmid_free_reserved(adev, AMDGPU_GFXHUB(0)); 2878 fpriv->vm.reserved_vmid[AMDGPU_GFXHUB(0)] = false; 2879 } 2880 break; 2881 default: 2882 return -EINVAL; 2883 } 2884 2885 return 0; 2886 } 2887 2888 /** 2889 * amdgpu_vm_handle_fault - graceful handling of VM faults. 2890 * @adev: amdgpu device pointer 2891 * @pasid: PASID of the VM 2892 * @ts: Timestamp of the fault 2893 * @vmid: VMID, only used for GFX 9.4.3. 2894 * @node_id: Node_id received in IH cookie. Only applicable for 2895 * GFX 9.4.3. 2896 * @addr: Address of the fault 2897 * @write_fault: true is write fault, false is read fault 2898 * 2899 * Try to gracefully handle a VM fault. Return true if the fault was handled and 2900 * shouldn't be reported any more. 2901 */ 2902 bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid, 2903 u32 vmid, u32 node_id, uint64_t addr, uint64_t ts, 2904 bool write_fault) 2905 { 2906 bool is_compute_context = false; 2907 struct amdgpu_bo *root; 2908 unsigned long irqflags; 2909 uint64_t value, flags; 2910 struct amdgpu_vm *vm; 2911 int r; 2912 2913 xa_lock_irqsave(&adev->vm_manager.pasids, irqflags); 2914 vm = xa_load(&adev->vm_manager.pasids, pasid); 2915 if (vm) { 2916 root = amdgpu_bo_ref(vm->root.bo); 2917 is_compute_context = vm->is_compute_context; 2918 } else { 2919 root = NULL; 2920 } 2921 xa_unlock_irqrestore(&adev->vm_manager.pasids, irqflags); 2922 2923 if (!root) 2924 return false; 2925 2926 addr /= AMDGPU_GPU_PAGE_SIZE; 2927 2928 if (is_compute_context && !svm_range_restore_pages(adev, pasid, vmid, 2929 node_id, addr, ts, write_fault)) { 2930 amdgpu_bo_unref(&root); 2931 return true; 2932 } 2933 2934 r = amdgpu_bo_reserve(root, true); 2935 if (r) 2936 goto error_unref; 2937 2938 /* Double check that the VM still exists */ 2939 xa_lock_irqsave(&adev->vm_manager.pasids, irqflags); 2940 vm = xa_load(&adev->vm_manager.pasids, pasid); 2941 if (vm && vm->root.bo != root) 2942 vm = NULL; 2943 xa_unlock_irqrestore(&adev->vm_manager.pasids, irqflags); 2944 if (!vm) 2945 goto error_unlock; 2946 2947 flags = AMDGPU_PTE_VALID | AMDGPU_PTE_SNOOPED | 2948 AMDGPU_PTE_SYSTEM; 2949 2950 if (is_compute_context) { 2951 /* Intentionally setting invalid PTE flag 2952 * combination to force a no-retry-fault 2953 */ 2954 flags = AMDGPU_VM_NORETRY_FLAGS; 2955 value = 0; 2956 } else if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_NEVER) { 2957 /* Redirect the access to the dummy page */ 2958 value = adev->dummy_page_addr; 2959 flags |= AMDGPU_PTE_EXECUTABLE | AMDGPU_PTE_READABLE | 2960 AMDGPU_PTE_WRITEABLE; 2961 2962 } else { 2963 /* Let the hw retry silently on the PTE */ 2964 value = 0; 2965 } 2966 2967 r = dma_resv_reserve_fences(root->tbo.base.resv, 1); 2968 if (r) { 2969 pr_debug("failed %d to reserve fence slot\n", r); 2970 goto error_unlock; 2971 } 2972 2973 r = amdgpu_vm_update_range(adev, vm, true, false, false, false, 2974 NULL, addr, addr, flags, value, 0, NULL, NULL, NULL); 2975 if (r) 2976 goto error_unlock; 2977 2978 r = amdgpu_vm_update_pdes(adev, vm, true); 2979 2980 error_unlock: 2981 amdgpu_bo_unreserve(root); 2982 if (r < 0) 2983 DRM_ERROR("Can't handle page fault (%d)\n", r); 2984 2985 error_unref: 2986 amdgpu_bo_unref(&root); 2987 2988 return false; 2989 } 2990 2991 #if defined(CONFIG_DEBUG_FS) 2992 /** 2993 * amdgpu_debugfs_vm_bo_info - print BO info for the VM 2994 * 2995 * @vm: Requested VM for printing BO info 2996 * @m: debugfs file 2997 * 2998 * Print BO information in debugfs file for the VM 2999 */ 3000 void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m) 3001 { 3002 struct amdgpu_bo_va *bo_va, *tmp; 3003 u64 total_idle = 0; 3004 u64 total_evicted = 0; 3005 u64 total_relocated = 0; 3006 u64 total_moved = 0; 3007 u64 total_invalidated = 0; 3008 u64 total_done = 0; 3009 unsigned int total_idle_objs = 0; 3010 unsigned int total_evicted_objs = 0; 3011 unsigned int total_relocated_objs = 0; 3012 unsigned int total_moved_objs = 0; 3013 unsigned int total_invalidated_objs = 0; 3014 unsigned int total_done_objs = 0; 3015 unsigned int id = 0; 3016 3017 spin_lock(&vm->status_lock); 3018 seq_puts(m, "\tIdle BOs:\n"); 3019 list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) { 3020 if (!bo_va->base.bo) 3021 continue; 3022 total_idle += amdgpu_bo_print_info(id++, bo_va->base.bo, m); 3023 } 3024 total_idle_objs = id; 3025 id = 0; 3026 3027 seq_puts(m, "\tEvicted BOs:\n"); 3028 list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status) { 3029 if (!bo_va->base.bo) 3030 continue; 3031 total_evicted += amdgpu_bo_print_info(id++, bo_va->base.bo, m); 3032 } 3033 total_evicted_objs = id; 3034 id = 0; 3035 3036 seq_puts(m, "\tRelocated BOs:\n"); 3037 list_for_each_entry_safe(bo_va, tmp, &vm->relocated, base.vm_status) { 3038 if (!bo_va->base.bo) 3039 continue; 3040 total_relocated += amdgpu_bo_print_info(id++, bo_va->base.bo, m); 3041 } 3042 total_relocated_objs = id; 3043 id = 0; 3044 3045 seq_puts(m, "\tMoved BOs:\n"); 3046 list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) { 3047 if (!bo_va->base.bo) 3048 continue; 3049 total_moved += amdgpu_bo_print_info(id++, bo_va->base.bo, m); 3050 } 3051 total_moved_objs = id; 3052 id = 0; 3053 3054 seq_puts(m, "\tInvalidated BOs:\n"); 3055 list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) { 3056 if (!bo_va->base.bo) 3057 continue; 3058 total_invalidated += amdgpu_bo_print_info(id++, bo_va->base.bo, m); 3059 } 3060 total_invalidated_objs = id; 3061 id = 0; 3062 3063 seq_puts(m, "\tDone BOs:\n"); 3064 list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status) { 3065 if (!bo_va->base.bo) 3066 continue; 3067 total_done += amdgpu_bo_print_info(id++, bo_va->base.bo, m); 3068 } 3069 spin_unlock(&vm->status_lock); 3070 total_done_objs = id; 3071 3072 seq_printf(m, "\tTotal idle size: %12lld\tobjs:\t%d\n", total_idle, 3073 total_idle_objs); 3074 seq_printf(m, "\tTotal evicted size: %12lld\tobjs:\t%d\n", total_evicted, 3075 total_evicted_objs); 3076 seq_printf(m, "\tTotal relocated size: %12lld\tobjs:\t%d\n", total_relocated, 3077 total_relocated_objs); 3078 seq_printf(m, "\tTotal moved size: %12lld\tobjs:\t%d\n", total_moved, 3079 total_moved_objs); 3080 seq_printf(m, "\tTotal invalidated size: %12lld\tobjs:\t%d\n", total_invalidated, 3081 total_invalidated_objs); 3082 seq_printf(m, "\tTotal done size: %12lld\tobjs:\t%d\n", total_done, 3083 total_done_objs); 3084 } 3085 #endif 3086 3087 /** 3088 * amdgpu_vm_update_fault_cache - update cached fault into. 3089 * @adev: amdgpu device pointer 3090 * @pasid: PASID of the VM 3091 * @addr: Address of the fault 3092 * @status: GPUVM fault status register 3093 * @vmhub: which vmhub got the fault 3094 * 3095 * Cache the fault info for later use by userspace in debugging. 3096 */ 3097 void amdgpu_vm_update_fault_cache(struct amdgpu_device *adev, 3098 unsigned int pasid, 3099 uint64_t addr, 3100 uint32_t status, 3101 unsigned int vmhub) 3102 { 3103 struct amdgpu_vm *vm; 3104 unsigned long flags; 3105 3106 xa_lock_irqsave(&adev->vm_manager.pasids, flags); 3107 3108 vm = xa_load(&adev->vm_manager.pasids, pasid); 3109 /* Don't update the fault cache if status is 0. In the multiple 3110 * fault case, subsequent faults will return a 0 status which is 3111 * useless for userspace and replaces the useful fault status, so 3112 * only update if status is non-0. 3113 */ 3114 if (vm && status) { 3115 vm->fault_info.addr = addr; 3116 vm->fault_info.status = status; 3117 /* 3118 * Update the fault information globally for later usage 3119 * when vm could be stale or freed. 3120 */ 3121 adev->vm_manager.fault_info.addr = addr; 3122 adev->vm_manager.fault_info.vmhub = vmhub; 3123 adev->vm_manager.fault_info.status = status; 3124 3125 if (AMDGPU_IS_GFXHUB(vmhub)) { 3126 vm->fault_info.vmhub = AMDGPU_VMHUB_TYPE_GFX; 3127 vm->fault_info.vmhub |= 3128 (vmhub - AMDGPU_GFXHUB_START) << AMDGPU_VMHUB_IDX_SHIFT; 3129 } else if (AMDGPU_IS_MMHUB0(vmhub)) { 3130 vm->fault_info.vmhub = AMDGPU_VMHUB_TYPE_MM0; 3131 vm->fault_info.vmhub |= 3132 (vmhub - AMDGPU_MMHUB0_START) << AMDGPU_VMHUB_IDX_SHIFT; 3133 } else if (AMDGPU_IS_MMHUB1(vmhub)) { 3134 vm->fault_info.vmhub = AMDGPU_VMHUB_TYPE_MM1; 3135 vm->fault_info.vmhub |= 3136 (vmhub - AMDGPU_MMHUB1_START) << AMDGPU_VMHUB_IDX_SHIFT; 3137 } else { 3138 WARN_ONCE(1, "Invalid vmhub %u\n", vmhub); 3139 } 3140 } 3141 xa_unlock_irqrestore(&adev->vm_manager.pasids, flags); 3142 } 3143 3144 /** 3145 * amdgpu_vm_is_bo_always_valid - check if the BO is VM always valid 3146 * 3147 * @vm: VM to test against. 3148 * @bo: BO to be tested. 3149 * 3150 * Returns true if the BO shares the dma_resv object with the root PD and is 3151 * always guaranteed to be valid inside the VM. 3152 */ 3153 bool amdgpu_vm_is_bo_always_valid(struct amdgpu_vm *vm, struct amdgpu_bo *bo) 3154 { 3155 return bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv; 3156 } 3157