1 /* 2 * Copyright 2009 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 /* 27 * Authors: 28 * Jerome Glisse <[email protected]> 29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> 30 * Dave Airlie 31 */ 32 #include <linux/list.h> 33 #include <linux/slab.h> 34 #include <linux/dma-buf.h> 35 36 #include <drm/drm_drv.h> 37 #include <drm/amdgpu_drm.h> 38 #include <drm/drm_cache.h> 39 #include "amdgpu.h" 40 #include "amdgpu_trace.h" 41 #include "amdgpu_amdkfd.h" 42 #include "amdgpu_vram_mgr.h" 43 44 /** 45 * DOC: amdgpu_object 46 * 47 * This defines the interfaces to operate on an &amdgpu_bo buffer object which 48 * represents memory used by driver (VRAM, system memory, etc.). The driver 49 * provides DRM/GEM APIs to userspace. DRM/GEM APIs then use these interfaces 50 * to create/destroy/set buffer object which are then managed by the kernel TTM 51 * memory manager. 52 * The interfaces are also used internally by kernel clients, including gfx, 53 * uvd, etc. for kernel managed allocations used by the GPU. 54 * 55 */ 56 57 static void amdgpu_bo_destroy(struct ttm_buffer_object *tbo) 58 { 59 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); 60 61 amdgpu_bo_kunmap(bo); 62 63 if (bo->tbo.base.import_attach) 64 drm_prime_gem_destroy(&bo->tbo.base, bo->tbo.sg); 65 drm_gem_object_release(&bo->tbo.base); 66 amdgpu_bo_unref(&bo->parent); 67 kvfree(bo); 68 } 69 70 static void amdgpu_bo_user_destroy(struct ttm_buffer_object *tbo) 71 { 72 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); 73 struct amdgpu_bo_user *ubo; 74 75 ubo = to_amdgpu_bo_user(bo); 76 kfree(ubo->metadata); 77 amdgpu_bo_destroy(tbo); 78 } 79 80 /** 81 * amdgpu_bo_is_amdgpu_bo - check if the buffer object is an &amdgpu_bo 82 * @bo: buffer object to be checked 83 * 84 * Uses destroy function associated with the object to determine if this is 85 * an &amdgpu_bo. 86 * 87 * Returns: 88 * true if the object belongs to &amdgpu_bo, false if not. 89 */ 90 bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo) 91 { 92 if (bo->destroy == &amdgpu_bo_destroy || 93 bo->destroy == &amdgpu_bo_user_destroy) 94 return true; 95 96 return false; 97 } 98 99 /** 100 * amdgpu_bo_placement_from_domain - set buffer's placement 101 * @abo: &amdgpu_bo buffer object whose placement is to be set 102 * @domain: requested domain 103 * 104 * Sets buffer's placement according to requested domain and the buffer's 105 * flags. 106 */ 107 void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain) 108 { 109 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); 110 struct ttm_placement *placement = &abo->placement; 111 struct ttm_place *places = abo->placements; 112 u64 flags = abo->flags; 113 u32 c = 0; 114 115 if (domain & AMDGPU_GEM_DOMAIN_VRAM) { 116 unsigned int visible_pfn = adev->gmc.visible_vram_size >> PAGE_SHIFT; 117 int8_t mem_id = KFD_XCP_MEM_ID(adev, abo->xcp_id); 118 119 if (adev->gmc.mem_partitions && mem_id >= 0) { 120 places[c].fpfn = adev->gmc.mem_partitions[mem_id].range.fpfn; 121 /* 122 * memory partition range lpfn is inclusive start + size - 1 123 * TTM place lpfn is exclusive start + size 124 */ 125 places[c].lpfn = adev->gmc.mem_partitions[mem_id].range.lpfn + 1; 126 } else { 127 places[c].fpfn = 0; 128 places[c].lpfn = 0; 129 } 130 places[c].mem_type = TTM_PL_VRAM; 131 places[c].flags = 0; 132 133 if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) 134 places[c].lpfn = min_not_zero(places[c].lpfn, visible_pfn); 135 else 136 places[c].flags |= TTM_PL_FLAG_TOPDOWN; 137 138 if (abo->tbo.type == ttm_bo_type_kernel && 139 flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) 140 places[c].flags |= TTM_PL_FLAG_CONTIGUOUS; 141 142 c++; 143 } 144 145 if (domain & AMDGPU_GEM_DOMAIN_DOORBELL) { 146 places[c].fpfn = 0; 147 places[c].lpfn = 0; 148 places[c].mem_type = AMDGPU_PL_DOORBELL; 149 places[c].flags = 0; 150 c++; 151 } 152 153 if (domain & AMDGPU_GEM_DOMAIN_GTT) { 154 places[c].fpfn = 0; 155 places[c].lpfn = 0; 156 places[c].mem_type = 157 abo->flags & AMDGPU_GEM_CREATE_PREEMPTIBLE ? 158 AMDGPU_PL_PREEMPT : TTM_PL_TT; 159 places[c].flags = 0; 160 /* 161 * When GTT is just an alternative to VRAM make sure that we 162 * only use it as fallback and still try to fill up VRAM first. 163 */ 164 if (domain & abo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) 165 places[c].flags |= TTM_PL_FLAG_FALLBACK; 166 c++; 167 } 168 169 if (domain & AMDGPU_GEM_DOMAIN_CPU) { 170 places[c].fpfn = 0; 171 places[c].lpfn = 0; 172 places[c].mem_type = TTM_PL_SYSTEM; 173 places[c].flags = 0; 174 c++; 175 } 176 177 if (domain & AMDGPU_GEM_DOMAIN_GDS) { 178 places[c].fpfn = 0; 179 places[c].lpfn = 0; 180 places[c].mem_type = AMDGPU_PL_GDS; 181 places[c].flags = 0; 182 c++; 183 } 184 185 if (domain & AMDGPU_GEM_DOMAIN_GWS) { 186 places[c].fpfn = 0; 187 places[c].lpfn = 0; 188 places[c].mem_type = AMDGPU_PL_GWS; 189 places[c].flags = 0; 190 c++; 191 } 192 193 if (domain & AMDGPU_GEM_DOMAIN_OA) { 194 places[c].fpfn = 0; 195 places[c].lpfn = 0; 196 places[c].mem_type = AMDGPU_PL_OA; 197 places[c].flags = 0; 198 c++; 199 } 200 201 if (!c) { 202 places[c].fpfn = 0; 203 places[c].lpfn = 0; 204 places[c].mem_type = TTM_PL_SYSTEM; 205 places[c].flags = 0; 206 c++; 207 } 208 209 BUG_ON(c > AMDGPU_BO_MAX_PLACEMENTS); 210 211 placement->num_placement = c; 212 placement->placement = places; 213 } 214 215 /** 216 * amdgpu_bo_create_reserved - create reserved BO for kernel use 217 * 218 * @adev: amdgpu device object 219 * @size: size for the new BO 220 * @align: alignment for the new BO 221 * @domain: where to place it 222 * @bo_ptr: used to initialize BOs in structures 223 * @gpu_addr: GPU addr of the pinned BO 224 * @cpu_addr: optional CPU address mapping 225 * 226 * Allocates and pins a BO for kernel internal use, and returns it still 227 * reserved. 228 * 229 * Note: For bo_ptr new BO is only created if bo_ptr points to NULL. 230 * 231 * Returns: 232 * 0 on success, negative error code otherwise. 233 */ 234 int amdgpu_bo_create_reserved(struct amdgpu_device *adev, 235 unsigned long size, int align, 236 u32 domain, struct amdgpu_bo **bo_ptr, 237 u64 *gpu_addr, void **cpu_addr) 238 { 239 struct amdgpu_bo_param bp; 240 bool free = false; 241 int r; 242 243 if (!size) { 244 amdgpu_bo_unref(bo_ptr); 245 return 0; 246 } 247 248 memset(&bp, 0, sizeof(bp)); 249 bp.size = size; 250 bp.byte_align = align; 251 bp.domain = domain; 252 bp.flags = cpu_addr ? AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED 253 : AMDGPU_GEM_CREATE_NO_CPU_ACCESS; 254 bp.flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; 255 bp.type = ttm_bo_type_kernel; 256 bp.resv = NULL; 257 bp.bo_ptr_size = sizeof(struct amdgpu_bo); 258 259 if (!*bo_ptr) { 260 r = amdgpu_bo_create(adev, &bp, bo_ptr); 261 if (r) { 262 dev_err(adev->dev, "(%d) failed to allocate kernel bo\n", 263 r); 264 return r; 265 } 266 free = true; 267 } 268 269 r = amdgpu_bo_reserve(*bo_ptr, false); 270 if (r) { 271 dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r); 272 goto error_free; 273 } 274 275 r = amdgpu_bo_pin(*bo_ptr, domain); 276 if (r) { 277 dev_err(adev->dev, "(%d) kernel bo pin failed\n", r); 278 goto error_unreserve; 279 } 280 281 r = amdgpu_ttm_alloc_gart(&(*bo_ptr)->tbo); 282 if (r) { 283 dev_err(adev->dev, "%p bind failed\n", *bo_ptr); 284 goto error_unpin; 285 } 286 287 if (gpu_addr) 288 *gpu_addr = amdgpu_bo_gpu_offset(*bo_ptr); 289 290 if (cpu_addr) { 291 r = amdgpu_bo_kmap(*bo_ptr, cpu_addr); 292 if (r) { 293 dev_err(adev->dev, "(%d) kernel bo map failed\n", r); 294 goto error_unpin; 295 } 296 } 297 298 return 0; 299 300 error_unpin: 301 amdgpu_bo_unpin(*bo_ptr); 302 error_unreserve: 303 amdgpu_bo_unreserve(*bo_ptr); 304 305 error_free: 306 if (free) 307 amdgpu_bo_unref(bo_ptr); 308 309 return r; 310 } 311 312 /** 313 * amdgpu_bo_create_kernel - create BO for kernel use 314 * 315 * @adev: amdgpu device object 316 * @size: size for the new BO 317 * @align: alignment for the new BO 318 * @domain: where to place it 319 * @bo_ptr: used to initialize BOs in structures 320 * @gpu_addr: GPU addr of the pinned BO 321 * @cpu_addr: optional CPU address mapping 322 * 323 * Allocates and pins a BO for kernel internal use. 324 * 325 * Note: For bo_ptr new BO is only created if bo_ptr points to NULL. 326 * 327 * Returns: 328 * 0 on success, negative error code otherwise. 329 */ 330 int amdgpu_bo_create_kernel(struct amdgpu_device *adev, 331 unsigned long size, int align, 332 u32 domain, struct amdgpu_bo **bo_ptr, 333 u64 *gpu_addr, void **cpu_addr) 334 { 335 int r; 336 337 r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr, 338 gpu_addr, cpu_addr); 339 340 if (r) 341 return r; 342 343 if (*bo_ptr) 344 amdgpu_bo_unreserve(*bo_ptr); 345 346 return 0; 347 } 348 349 /** 350 * amdgpu_bo_create_kernel_at - create BO for kernel use at specific location 351 * 352 * @adev: amdgpu device object 353 * @offset: offset of the BO 354 * @size: size of the BO 355 * @bo_ptr: used to initialize BOs in structures 356 * @cpu_addr: optional CPU address mapping 357 * 358 * Creates a kernel BO at a specific offset in VRAM. 359 * 360 * Returns: 361 * 0 on success, negative error code otherwise. 362 */ 363 int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev, 364 uint64_t offset, uint64_t size, 365 struct amdgpu_bo **bo_ptr, void **cpu_addr) 366 { 367 struct ttm_operation_ctx ctx = { false, false }; 368 unsigned int i; 369 int r; 370 371 offset &= PAGE_MASK; 372 size = ALIGN(size, PAGE_SIZE); 373 374 r = amdgpu_bo_create_reserved(adev, size, PAGE_SIZE, 375 AMDGPU_GEM_DOMAIN_VRAM, bo_ptr, NULL, 376 cpu_addr); 377 if (r) 378 return r; 379 380 if ((*bo_ptr) == NULL) 381 return 0; 382 383 /* 384 * Remove the original mem node and create a new one at the request 385 * position. 386 */ 387 if (cpu_addr) 388 amdgpu_bo_kunmap(*bo_ptr); 389 390 ttm_resource_free(&(*bo_ptr)->tbo, &(*bo_ptr)->tbo.resource); 391 392 for (i = 0; i < (*bo_ptr)->placement.num_placement; ++i) { 393 (*bo_ptr)->placements[i].fpfn = offset >> PAGE_SHIFT; 394 (*bo_ptr)->placements[i].lpfn = (offset + size) >> PAGE_SHIFT; 395 } 396 r = ttm_bo_mem_space(&(*bo_ptr)->tbo, &(*bo_ptr)->placement, 397 &(*bo_ptr)->tbo.resource, &ctx); 398 if (r) 399 goto error; 400 401 if (cpu_addr) { 402 r = amdgpu_bo_kmap(*bo_ptr, cpu_addr); 403 if (r) 404 goto error; 405 } 406 407 amdgpu_bo_unreserve(*bo_ptr); 408 return 0; 409 410 error: 411 amdgpu_bo_unreserve(*bo_ptr); 412 amdgpu_bo_unref(bo_ptr); 413 return r; 414 } 415 416 /** 417 * amdgpu_bo_free_kernel - free BO for kernel use 418 * 419 * @bo: amdgpu BO to free 420 * @gpu_addr: pointer to where the BO's GPU memory space address was stored 421 * @cpu_addr: pointer to where the BO's CPU memory space address was stored 422 * 423 * unmaps and unpin a BO for kernel internal use. 424 */ 425 void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr, 426 void **cpu_addr) 427 { 428 if (*bo == NULL) 429 return; 430 431 WARN_ON(amdgpu_ttm_adev((*bo)->tbo.bdev)->in_suspend); 432 433 if (likely(amdgpu_bo_reserve(*bo, true) == 0)) { 434 if (cpu_addr) 435 amdgpu_bo_kunmap(*bo); 436 437 amdgpu_bo_unpin(*bo); 438 amdgpu_bo_unreserve(*bo); 439 } 440 amdgpu_bo_unref(bo); 441 442 if (gpu_addr) 443 *gpu_addr = 0; 444 445 if (cpu_addr) 446 *cpu_addr = NULL; 447 } 448 449 /* Validate bo size is bit bigger than the request domain */ 450 static bool amdgpu_bo_validate_size(struct amdgpu_device *adev, 451 unsigned long size, u32 domain) 452 { 453 struct ttm_resource_manager *man = NULL; 454 455 /* 456 * If GTT is part of requested domains the check must succeed to 457 * allow fall back to GTT. 458 */ 459 if (domain & AMDGPU_GEM_DOMAIN_GTT) 460 man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT); 461 else if (domain & AMDGPU_GEM_DOMAIN_VRAM) 462 man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM); 463 else 464 return true; 465 466 if (!man) { 467 if (domain & AMDGPU_GEM_DOMAIN_GTT) 468 WARN_ON_ONCE("GTT domain requested but GTT mem manager uninitialized"); 469 return false; 470 } 471 472 /* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU, _DOMAIN_DOORBELL */ 473 if (size < man->size) 474 return true; 475 476 DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size, man->size); 477 return false; 478 } 479 480 bool amdgpu_bo_support_uswc(u64 bo_flags) 481 { 482 483 #ifdef CONFIG_X86_32 484 /* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit 485 * See https://bugs.freedesktop.org/show_bug.cgi?id=84627 486 */ 487 return false; 488 #elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT) 489 /* Don't try to enable write-combining when it can't work, or things 490 * may be slow 491 * See https://bugs.freedesktop.org/show_bug.cgi?id=88758 492 */ 493 494 #ifndef CONFIG_COMPILE_TEST 495 #warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \ 496 thanks to write-combining 497 #endif 498 499 if (bo_flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) 500 DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for " 501 "better performance thanks to write-combining\n"); 502 return false; 503 #else 504 /* For architectures that don't support WC memory, 505 * mask out the WC flag from the BO 506 */ 507 if (!drm_arch_can_wc_memory()) 508 return false; 509 510 return true; 511 #endif 512 } 513 514 /** 515 * amdgpu_bo_create - create an &amdgpu_bo buffer object 516 * @adev: amdgpu device object 517 * @bp: parameters to be used for the buffer object 518 * @bo_ptr: pointer to the buffer object pointer 519 * 520 * Creates an &amdgpu_bo buffer object. 521 * 522 * Returns: 523 * 0 for success or a negative error code on failure. 524 */ 525 int amdgpu_bo_create(struct amdgpu_device *adev, 526 struct amdgpu_bo_param *bp, 527 struct amdgpu_bo **bo_ptr) 528 { 529 struct ttm_operation_ctx ctx = { 530 .interruptible = (bp->type != ttm_bo_type_kernel), 531 .no_wait_gpu = bp->no_wait_gpu, 532 /* We opt to avoid OOM on system pages allocations */ 533 .gfp_retry_mayfail = true, 534 .allow_res_evict = bp->type != ttm_bo_type_kernel, 535 .resv = bp->resv 536 }; 537 struct amdgpu_bo *bo; 538 unsigned long page_align, size = bp->size; 539 int r; 540 541 /* Note that GDS/GWS/OA allocates 1 page per byte/resource. */ 542 if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) { 543 /* GWS and OA don't need any alignment. */ 544 page_align = bp->byte_align; 545 size <<= PAGE_SHIFT; 546 547 } else if (bp->domain & AMDGPU_GEM_DOMAIN_GDS) { 548 /* Both size and alignment must be a multiple of 4. */ 549 page_align = ALIGN(bp->byte_align, 4); 550 size = ALIGN(size, 4) << PAGE_SHIFT; 551 } else { 552 /* Memory should be aligned at least to a page size. */ 553 page_align = ALIGN(bp->byte_align, PAGE_SIZE) >> PAGE_SHIFT; 554 size = ALIGN(size, PAGE_SIZE); 555 } 556 557 if (!amdgpu_bo_validate_size(adev, size, bp->domain)) 558 return -ENOMEM; 559 560 BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo)); 561 562 *bo_ptr = NULL; 563 bo = kvzalloc(bp->bo_ptr_size, GFP_KERNEL); 564 if (bo == NULL) 565 return -ENOMEM; 566 drm_gem_private_object_init(adev_to_drm(adev), &bo->tbo.base, size); 567 bo->vm_bo = NULL; 568 bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain : 569 bp->domain; 570 bo->allowed_domains = bo->preferred_domains; 571 if (bp->type != ttm_bo_type_kernel && 572 !(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE) && 573 bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM) 574 bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT; 575 576 bo->flags = bp->flags; 577 578 if (adev->gmc.mem_partitions) 579 /* For GPUs with spatial partitioning, bo->xcp_id=-1 means any partition */ 580 bo->xcp_id = bp->xcp_id_plus1 - 1; 581 else 582 /* For GPUs without spatial partitioning */ 583 bo->xcp_id = 0; 584 585 if (!amdgpu_bo_support_uswc(bo->flags)) 586 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; 587 588 bo->tbo.bdev = &adev->mman.bdev; 589 if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA | 590 AMDGPU_GEM_DOMAIN_GDS)) 591 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); 592 else 593 amdgpu_bo_placement_from_domain(bo, bp->domain); 594 if (bp->type == ttm_bo_type_kernel) 595 bo->tbo.priority = 2; 596 else if (!(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE)) 597 bo->tbo.priority = 1; 598 599 if (!bp->destroy) 600 bp->destroy = &amdgpu_bo_destroy; 601 602 r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, bp->type, 603 &bo->placement, page_align, &ctx, NULL, 604 bp->resv, bp->destroy); 605 if (unlikely(r != 0)) 606 return r; 607 608 if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && 609 amdgpu_res_cpu_visible(adev, bo->tbo.resource)) 610 amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 611 ctx.bytes_moved); 612 else 613 amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0); 614 615 if (bp->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED && 616 bo->tbo.resource->mem_type == TTM_PL_VRAM) { 617 struct dma_fence *fence; 618 619 r = amdgpu_ttm_clear_buffer(bo, bo->tbo.base.resv, &fence); 620 if (unlikely(r)) 621 goto fail_unreserve; 622 623 dma_resv_add_fence(bo->tbo.base.resv, fence, 624 DMA_RESV_USAGE_KERNEL); 625 dma_fence_put(fence); 626 } 627 if (!bp->resv) 628 amdgpu_bo_unreserve(bo); 629 *bo_ptr = bo; 630 631 trace_amdgpu_bo_create(bo); 632 633 /* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */ 634 if (bp->type == ttm_bo_type_device) 635 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 636 637 return 0; 638 639 fail_unreserve: 640 if (!bp->resv) 641 dma_resv_unlock(bo->tbo.base.resv); 642 amdgpu_bo_unref(&bo); 643 return r; 644 } 645 646 /** 647 * amdgpu_bo_create_user - create an &amdgpu_bo_user buffer object 648 * @adev: amdgpu device object 649 * @bp: parameters to be used for the buffer object 650 * @ubo_ptr: pointer to the buffer object pointer 651 * 652 * Create a BO to be used by user application; 653 * 654 * Returns: 655 * 0 for success or a negative error code on failure. 656 */ 657 658 int amdgpu_bo_create_user(struct amdgpu_device *adev, 659 struct amdgpu_bo_param *bp, 660 struct amdgpu_bo_user **ubo_ptr) 661 { 662 struct amdgpu_bo *bo_ptr; 663 int r; 664 665 bp->bo_ptr_size = sizeof(struct amdgpu_bo_user); 666 bp->destroy = &amdgpu_bo_user_destroy; 667 r = amdgpu_bo_create(adev, bp, &bo_ptr); 668 if (r) 669 return r; 670 671 *ubo_ptr = to_amdgpu_bo_user(bo_ptr); 672 return r; 673 } 674 675 /** 676 * amdgpu_bo_create_vm - create an &amdgpu_bo_vm buffer object 677 * @adev: amdgpu device object 678 * @bp: parameters to be used for the buffer object 679 * @vmbo_ptr: pointer to the buffer object pointer 680 * 681 * Create a BO to be for GPUVM. 682 * 683 * Returns: 684 * 0 for success or a negative error code on failure. 685 */ 686 687 int amdgpu_bo_create_vm(struct amdgpu_device *adev, 688 struct amdgpu_bo_param *bp, 689 struct amdgpu_bo_vm **vmbo_ptr) 690 { 691 struct amdgpu_bo *bo_ptr; 692 int r; 693 694 /* bo_ptr_size will be determined by the caller and it depends on 695 * num of amdgpu_vm_pt entries. 696 */ 697 BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo_vm)); 698 r = amdgpu_bo_create(adev, bp, &bo_ptr); 699 if (r) 700 return r; 701 702 *vmbo_ptr = to_amdgpu_bo_vm(bo_ptr); 703 return r; 704 } 705 706 /** 707 * amdgpu_bo_kmap - map an &amdgpu_bo buffer object 708 * @bo: &amdgpu_bo buffer object to be mapped 709 * @ptr: kernel virtual address to be returned 710 * 711 * Calls ttm_bo_kmap() to set up the kernel virtual mapping; calls 712 * amdgpu_bo_kptr() to get the kernel virtual address. 713 * 714 * Returns: 715 * 0 for success or a negative error code on failure. 716 */ 717 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr) 718 { 719 void *kptr; 720 long r; 721 722 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) 723 return -EPERM; 724 725 r = dma_resv_wait_timeout(bo->tbo.base.resv, DMA_RESV_USAGE_KERNEL, 726 false, MAX_SCHEDULE_TIMEOUT); 727 if (r < 0) 728 return r; 729 730 kptr = amdgpu_bo_kptr(bo); 731 if (kptr) { 732 if (ptr) 733 *ptr = kptr; 734 return 0; 735 } 736 737 r = ttm_bo_kmap(&bo->tbo, 0, PFN_UP(bo->tbo.base.size), &bo->kmap); 738 if (r) 739 return r; 740 741 if (ptr) 742 *ptr = amdgpu_bo_kptr(bo); 743 744 return 0; 745 } 746 747 /** 748 * amdgpu_bo_kptr - returns a kernel virtual address of the buffer object 749 * @bo: &amdgpu_bo buffer object 750 * 751 * Calls ttm_kmap_obj_virtual() to get the kernel virtual address 752 * 753 * Returns: 754 * the virtual address of a buffer object area. 755 */ 756 void *amdgpu_bo_kptr(struct amdgpu_bo *bo) 757 { 758 bool is_iomem; 759 760 return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem); 761 } 762 763 /** 764 * amdgpu_bo_kunmap - unmap an &amdgpu_bo buffer object 765 * @bo: &amdgpu_bo buffer object to be unmapped 766 * 767 * Unmaps a kernel map set up by amdgpu_bo_kmap(). 768 */ 769 void amdgpu_bo_kunmap(struct amdgpu_bo *bo) 770 { 771 if (bo->kmap.bo) 772 ttm_bo_kunmap(&bo->kmap); 773 } 774 775 /** 776 * amdgpu_bo_ref - reference an &amdgpu_bo buffer object 777 * @bo: &amdgpu_bo buffer object 778 * 779 * References the contained &ttm_buffer_object. 780 * 781 * Returns: 782 * a refcounted pointer to the &amdgpu_bo buffer object. 783 */ 784 struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo) 785 { 786 if (bo == NULL) 787 return NULL; 788 789 ttm_bo_get(&bo->tbo); 790 return bo; 791 } 792 793 /** 794 * amdgpu_bo_unref - unreference an &amdgpu_bo buffer object 795 * @bo: &amdgpu_bo buffer object 796 * 797 * Unreferences the contained &ttm_buffer_object and clear the pointer 798 */ 799 void amdgpu_bo_unref(struct amdgpu_bo **bo) 800 { 801 struct ttm_buffer_object *tbo; 802 803 if ((*bo) == NULL) 804 return; 805 806 tbo = &((*bo)->tbo); 807 ttm_bo_put(tbo); 808 *bo = NULL; 809 } 810 811 /** 812 * amdgpu_bo_pin - pin an &amdgpu_bo buffer object 813 * @bo: &amdgpu_bo buffer object to be pinned 814 * @domain: domain to be pinned to 815 * 816 * Pins the buffer object according to requested domain. If the memory is 817 * unbound gart memory, binds the pages into gart table. Adjusts pin_count and 818 * pin_size accordingly. 819 * 820 * Pinning means to lock pages in memory along with keeping them at a fixed 821 * offset. It is required when a buffer can not be moved, for example, when 822 * a display buffer is being scanned out. 823 * 824 * Returns: 825 * 0 for success or a negative error code on failure. 826 */ 827 int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain) 828 { 829 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 830 struct ttm_operation_ctx ctx = { false, false }; 831 int r, i; 832 833 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) 834 return -EPERM; 835 836 /* Check domain to be pinned to against preferred domains */ 837 if (bo->preferred_domains & domain) 838 domain = bo->preferred_domains & domain; 839 840 /* A shared bo cannot be migrated to VRAM */ 841 if (bo->tbo.base.import_attach) { 842 if (domain & AMDGPU_GEM_DOMAIN_GTT) 843 domain = AMDGPU_GEM_DOMAIN_GTT; 844 else 845 return -EINVAL; 846 } 847 848 if (bo->tbo.pin_count) { 849 uint32_t mem_type = bo->tbo.resource->mem_type; 850 uint32_t mem_flags = bo->tbo.resource->placement; 851 852 if (!(domain & amdgpu_mem_type_to_domain(mem_type))) 853 return -EINVAL; 854 855 if ((mem_type == TTM_PL_VRAM) && 856 (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) && 857 !(mem_flags & TTM_PL_FLAG_CONTIGUOUS)) 858 return -EINVAL; 859 860 ttm_bo_pin(&bo->tbo); 861 return 0; 862 } 863 864 /* This assumes only APU display buffers are pinned with (VRAM|GTT). 865 * See function amdgpu_display_supported_domains() 866 */ 867 domain = amdgpu_bo_get_preferred_domain(adev, domain); 868 869 if (bo->tbo.base.import_attach) 870 dma_buf_pin(bo->tbo.base.import_attach); 871 872 /* force to pin into visible video ram */ 873 if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) 874 bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 875 amdgpu_bo_placement_from_domain(bo, domain); 876 for (i = 0; i < bo->placement.num_placement; i++) { 877 if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS && 878 bo->placements[i].mem_type == TTM_PL_VRAM) 879 bo->placements[i].flags |= TTM_PL_FLAG_CONTIGUOUS; 880 } 881 882 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 883 if (unlikely(r)) { 884 dev_err(adev->dev, "%p pin failed\n", bo); 885 goto error; 886 } 887 888 ttm_bo_pin(&bo->tbo); 889 890 if (bo->tbo.resource->mem_type == TTM_PL_VRAM) { 891 atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size); 892 atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo), 893 &adev->visible_pin_size); 894 } else if (bo->tbo.resource->mem_type == TTM_PL_TT) { 895 atomic64_add(amdgpu_bo_size(bo), &adev->gart_pin_size); 896 } 897 898 error: 899 return r; 900 } 901 902 /** 903 * amdgpu_bo_unpin - unpin an &amdgpu_bo buffer object 904 * @bo: &amdgpu_bo buffer object to be unpinned 905 * 906 * Decreases the pin_count, and clears the flags if pin_count reaches 0. 907 * Changes placement and pin size accordingly. 908 * 909 * Returns: 910 * 0 for success or a negative error code on failure. 911 */ 912 void amdgpu_bo_unpin(struct amdgpu_bo *bo) 913 { 914 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 915 916 ttm_bo_unpin(&bo->tbo); 917 if (bo->tbo.pin_count) 918 return; 919 920 if (bo->tbo.base.import_attach) 921 dma_buf_unpin(bo->tbo.base.import_attach); 922 923 if (bo->tbo.resource->mem_type == TTM_PL_VRAM) { 924 atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size); 925 atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo), 926 &adev->visible_pin_size); 927 } else if (bo->tbo.resource->mem_type == TTM_PL_TT) { 928 atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size); 929 } 930 931 } 932 933 static const char * const amdgpu_vram_names[] = { 934 "UNKNOWN", 935 "GDDR1", 936 "DDR2", 937 "GDDR3", 938 "GDDR4", 939 "GDDR5", 940 "HBM", 941 "DDR3", 942 "DDR4", 943 "GDDR6", 944 "DDR5", 945 "LPDDR4", 946 "LPDDR5" 947 }; 948 949 /** 950 * amdgpu_bo_init - initialize memory manager 951 * @adev: amdgpu device object 952 * 953 * Calls amdgpu_ttm_init() to initialize amdgpu memory manager. 954 * 955 * Returns: 956 * 0 for success or a negative error code on failure. 957 */ 958 int amdgpu_bo_init(struct amdgpu_device *adev) 959 { 960 /* On A+A platform, VRAM can be mapped as WB */ 961 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { 962 /* reserve PAT memory space to WC for VRAM */ 963 int r = arch_io_reserve_memtype_wc(adev->gmc.aper_base, 964 adev->gmc.aper_size); 965 966 if (r) { 967 DRM_ERROR("Unable to set WC memtype for the aperture base\n"); 968 return r; 969 } 970 971 /* Add an MTRR for the VRAM */ 972 adev->gmc.vram_mtrr = arch_phys_wc_add(adev->gmc.aper_base, 973 adev->gmc.aper_size); 974 } 975 976 DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n", 977 adev->gmc.mc_vram_size >> 20, 978 (unsigned long long)adev->gmc.aper_size >> 20); 979 DRM_INFO("RAM width %dbits %s\n", 980 adev->gmc.vram_width, amdgpu_vram_names[adev->gmc.vram_type]); 981 return amdgpu_ttm_init(adev); 982 } 983 984 /** 985 * amdgpu_bo_fini - tear down memory manager 986 * @adev: amdgpu device object 987 * 988 * Reverses amdgpu_bo_init() to tear down memory manager. 989 */ 990 void amdgpu_bo_fini(struct amdgpu_device *adev) 991 { 992 int idx; 993 994 amdgpu_ttm_fini(adev); 995 996 if (drm_dev_enter(adev_to_drm(adev), &idx)) { 997 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { 998 arch_phys_wc_del(adev->gmc.vram_mtrr); 999 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); 1000 } 1001 drm_dev_exit(idx); 1002 } 1003 } 1004 1005 /** 1006 * amdgpu_bo_set_tiling_flags - set tiling flags 1007 * @bo: &amdgpu_bo buffer object 1008 * @tiling_flags: new flags 1009 * 1010 * Sets buffer object's tiling flags with the new one. Used by GEM ioctl or 1011 * kernel driver to set the tiling flags on a buffer. 1012 * 1013 * Returns: 1014 * 0 for success or a negative error code on failure. 1015 */ 1016 int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags) 1017 { 1018 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1019 struct amdgpu_bo_user *ubo; 1020 1021 BUG_ON(bo->tbo.type == ttm_bo_type_kernel); 1022 if (adev->family <= AMDGPU_FAMILY_CZ && 1023 AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6) 1024 return -EINVAL; 1025 1026 ubo = to_amdgpu_bo_user(bo); 1027 ubo->tiling_flags = tiling_flags; 1028 return 0; 1029 } 1030 1031 /** 1032 * amdgpu_bo_get_tiling_flags - get tiling flags 1033 * @bo: &amdgpu_bo buffer object 1034 * @tiling_flags: returned flags 1035 * 1036 * Gets buffer object's tiling flags. Used by GEM ioctl or kernel driver to 1037 * set the tiling flags on a buffer. 1038 */ 1039 void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags) 1040 { 1041 struct amdgpu_bo_user *ubo; 1042 1043 BUG_ON(bo->tbo.type == ttm_bo_type_kernel); 1044 dma_resv_assert_held(bo->tbo.base.resv); 1045 ubo = to_amdgpu_bo_user(bo); 1046 1047 if (tiling_flags) 1048 *tiling_flags = ubo->tiling_flags; 1049 } 1050 1051 /** 1052 * amdgpu_bo_set_metadata - set metadata 1053 * @bo: &amdgpu_bo buffer object 1054 * @metadata: new metadata 1055 * @metadata_size: size of the new metadata 1056 * @flags: flags of the new metadata 1057 * 1058 * Sets buffer object's metadata, its size and flags. 1059 * Used via GEM ioctl. 1060 * 1061 * Returns: 1062 * 0 for success or a negative error code on failure. 1063 */ 1064 int amdgpu_bo_set_metadata(struct amdgpu_bo *bo, void *metadata, 1065 u32 metadata_size, uint64_t flags) 1066 { 1067 struct amdgpu_bo_user *ubo; 1068 void *buffer; 1069 1070 BUG_ON(bo->tbo.type == ttm_bo_type_kernel); 1071 ubo = to_amdgpu_bo_user(bo); 1072 if (!metadata_size) { 1073 if (ubo->metadata_size) { 1074 kfree(ubo->metadata); 1075 ubo->metadata = NULL; 1076 ubo->metadata_size = 0; 1077 } 1078 return 0; 1079 } 1080 1081 if (metadata == NULL) 1082 return -EINVAL; 1083 1084 buffer = kmemdup(metadata, metadata_size, GFP_KERNEL); 1085 if (buffer == NULL) 1086 return -ENOMEM; 1087 1088 kfree(ubo->metadata); 1089 ubo->metadata_flags = flags; 1090 ubo->metadata = buffer; 1091 ubo->metadata_size = metadata_size; 1092 1093 return 0; 1094 } 1095 1096 /** 1097 * amdgpu_bo_get_metadata - get metadata 1098 * @bo: &amdgpu_bo buffer object 1099 * @buffer: returned metadata 1100 * @buffer_size: size of the buffer 1101 * @metadata_size: size of the returned metadata 1102 * @flags: flags of the returned metadata 1103 * 1104 * Gets buffer object's metadata, its size and flags. buffer_size shall not be 1105 * less than metadata_size. 1106 * Used via GEM ioctl. 1107 * 1108 * Returns: 1109 * 0 for success or a negative error code on failure. 1110 */ 1111 int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer, 1112 size_t buffer_size, uint32_t *metadata_size, 1113 uint64_t *flags) 1114 { 1115 struct amdgpu_bo_user *ubo; 1116 1117 if (!buffer && !metadata_size) 1118 return -EINVAL; 1119 1120 BUG_ON(bo->tbo.type == ttm_bo_type_kernel); 1121 ubo = to_amdgpu_bo_user(bo); 1122 if (metadata_size) 1123 *metadata_size = ubo->metadata_size; 1124 1125 if (buffer) { 1126 if (buffer_size < ubo->metadata_size) 1127 return -EINVAL; 1128 1129 if (ubo->metadata_size) 1130 memcpy(buffer, ubo->metadata, ubo->metadata_size); 1131 } 1132 1133 if (flags) 1134 *flags = ubo->metadata_flags; 1135 1136 return 0; 1137 } 1138 1139 /** 1140 * amdgpu_bo_move_notify - notification about a memory move 1141 * @bo: pointer to a buffer object 1142 * @evict: if this move is evicting the buffer from the graphics address space 1143 * @new_mem: new resource for backing the BO 1144 * 1145 * Marks the corresponding &amdgpu_bo buffer object as invalid, also performs 1146 * bookkeeping. 1147 * TTM driver callback which is called when ttm moves a buffer. 1148 */ 1149 void amdgpu_bo_move_notify(struct ttm_buffer_object *bo, 1150 bool evict, 1151 struct ttm_resource *new_mem) 1152 { 1153 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 1154 struct ttm_resource *old_mem = bo->resource; 1155 struct amdgpu_bo *abo; 1156 1157 if (!amdgpu_bo_is_amdgpu_bo(bo)) 1158 return; 1159 1160 abo = ttm_to_amdgpu_bo(bo); 1161 amdgpu_vm_bo_invalidate(adev, abo, evict); 1162 1163 amdgpu_bo_kunmap(abo); 1164 1165 if (abo->tbo.base.dma_buf && !abo->tbo.base.import_attach && 1166 old_mem && old_mem->mem_type != TTM_PL_SYSTEM) 1167 dma_buf_move_notify(abo->tbo.base.dma_buf); 1168 1169 /* move_notify is called before move happens */ 1170 trace_amdgpu_bo_move(abo, new_mem ? new_mem->mem_type : -1, 1171 old_mem ? old_mem->mem_type : -1); 1172 } 1173 1174 void amdgpu_bo_get_memory(struct amdgpu_bo *bo, 1175 struct amdgpu_mem_stats *stats) 1176 { 1177 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1178 struct ttm_resource *res = bo->tbo.resource; 1179 uint64_t size = amdgpu_bo_size(bo); 1180 struct drm_gem_object *obj; 1181 bool shared; 1182 1183 /* Abort if the BO doesn't currently have a backing store */ 1184 if (!res) 1185 return; 1186 1187 obj = &bo->tbo.base; 1188 shared = drm_gem_object_is_shared_for_memory_stats(obj); 1189 1190 switch (res->mem_type) { 1191 case TTM_PL_VRAM: 1192 stats->vram += size; 1193 if (amdgpu_res_cpu_visible(adev, res)) 1194 stats->visible_vram += size; 1195 if (shared) 1196 stats->vram_shared += size; 1197 break; 1198 case TTM_PL_TT: 1199 stats->gtt += size; 1200 if (shared) 1201 stats->gtt_shared += size; 1202 break; 1203 case TTM_PL_SYSTEM: 1204 default: 1205 stats->cpu += size; 1206 if (shared) 1207 stats->cpu_shared += size; 1208 break; 1209 } 1210 1211 if (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) { 1212 stats->requested_vram += size; 1213 if (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) 1214 stats->requested_visible_vram += size; 1215 1216 if (res->mem_type != TTM_PL_VRAM) { 1217 stats->evicted_vram += size; 1218 if (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) 1219 stats->evicted_visible_vram += size; 1220 } 1221 } else if (bo->preferred_domains & AMDGPU_GEM_DOMAIN_GTT) { 1222 stats->requested_gtt += size; 1223 } 1224 } 1225 1226 /** 1227 * amdgpu_bo_release_notify - notification about a BO being released 1228 * @bo: pointer to a buffer object 1229 * 1230 * Wipes VRAM buffers whose contents should not be leaked before the 1231 * memory is released. 1232 */ 1233 void amdgpu_bo_release_notify(struct ttm_buffer_object *bo) 1234 { 1235 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 1236 struct dma_fence *fence = NULL; 1237 struct amdgpu_bo *abo; 1238 int r; 1239 1240 if (!amdgpu_bo_is_amdgpu_bo(bo)) 1241 return; 1242 1243 abo = ttm_to_amdgpu_bo(bo); 1244 1245 WARN_ON(abo->vm_bo); 1246 1247 if (abo->kfd_bo) 1248 amdgpu_amdkfd_release_notify(abo); 1249 1250 /* We only remove the fence if the resv has individualized. */ 1251 WARN_ON_ONCE(bo->type == ttm_bo_type_kernel 1252 && bo->base.resv != &bo->base._resv); 1253 if (bo->base.resv == &bo->base._resv) 1254 amdgpu_amdkfd_remove_fence_on_pt_pd_bos(abo); 1255 1256 if (!bo->resource || bo->resource->mem_type != TTM_PL_VRAM || 1257 !(abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE) || 1258 adev->in_suspend || drm_dev_is_unplugged(adev_to_drm(adev))) 1259 return; 1260 1261 if (WARN_ON_ONCE(!dma_resv_trylock(bo->base.resv))) 1262 return; 1263 1264 r = amdgpu_fill_buffer(abo, 0, bo->base.resv, &fence, true); 1265 if (!WARN_ON(r)) { 1266 amdgpu_vram_mgr_set_cleared(bo->resource); 1267 amdgpu_bo_fence(abo, fence, false); 1268 dma_fence_put(fence); 1269 } 1270 1271 dma_resv_unlock(bo->base.resv); 1272 } 1273 1274 /** 1275 * amdgpu_bo_fault_reserve_notify - notification about a memory fault 1276 * @bo: pointer to a buffer object 1277 * 1278 * Notifies the driver we are taking a fault on this BO and have reserved it, 1279 * also performs bookkeeping. 1280 * TTM driver callback for dealing with vm faults. 1281 * 1282 * Returns: 1283 * 0 for success or a negative error code on failure. 1284 */ 1285 vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo) 1286 { 1287 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 1288 struct ttm_operation_ctx ctx = { false, false }; 1289 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo); 1290 int r; 1291 1292 /* Remember that this BO was accessed by the CPU */ 1293 abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 1294 1295 if (amdgpu_res_cpu_visible(adev, bo->resource)) 1296 return 0; 1297 1298 /* Can't move a pinned BO to visible VRAM */ 1299 if (abo->tbo.pin_count > 0) 1300 return VM_FAULT_SIGBUS; 1301 1302 /* hurrah the memory is not visible ! */ 1303 atomic64_inc(&adev->num_vram_cpu_page_faults); 1304 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM | 1305 AMDGPU_GEM_DOMAIN_GTT); 1306 1307 /* Avoid costly evictions; only set GTT as a busy placement */ 1308 abo->placements[0].flags |= TTM_PL_FLAG_DESIRED; 1309 1310 r = ttm_bo_validate(bo, &abo->placement, &ctx); 1311 if (unlikely(r == -EBUSY || r == -ERESTARTSYS)) 1312 return VM_FAULT_NOPAGE; 1313 else if (unlikely(r)) 1314 return VM_FAULT_SIGBUS; 1315 1316 /* this should never happen */ 1317 if (bo->resource->mem_type == TTM_PL_VRAM && 1318 !amdgpu_res_cpu_visible(adev, bo->resource)) 1319 return VM_FAULT_SIGBUS; 1320 1321 ttm_bo_move_to_lru_tail_unlocked(bo); 1322 return 0; 1323 } 1324 1325 /** 1326 * amdgpu_bo_fence - add fence to buffer object 1327 * 1328 * @bo: buffer object in question 1329 * @fence: fence to add 1330 * @shared: true if fence should be added shared 1331 * 1332 */ 1333 void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence, 1334 bool shared) 1335 { 1336 struct dma_resv *resv = bo->tbo.base.resv; 1337 int r; 1338 1339 r = dma_resv_reserve_fences(resv, 1); 1340 if (r) { 1341 /* As last resort on OOM we block for the fence */ 1342 dma_fence_wait(fence, false); 1343 return; 1344 } 1345 1346 dma_resv_add_fence(resv, fence, shared ? DMA_RESV_USAGE_READ : 1347 DMA_RESV_USAGE_WRITE); 1348 } 1349 1350 /** 1351 * amdgpu_bo_sync_wait_resv - Wait for BO reservation fences 1352 * 1353 * @adev: amdgpu device pointer 1354 * @resv: reservation object to sync to 1355 * @sync_mode: synchronization mode 1356 * @owner: fence owner 1357 * @intr: Whether the wait is interruptible 1358 * 1359 * Extract the fences from the reservation object and waits for them to finish. 1360 * 1361 * Returns: 1362 * 0 on success, errno otherwise. 1363 */ 1364 int amdgpu_bo_sync_wait_resv(struct amdgpu_device *adev, struct dma_resv *resv, 1365 enum amdgpu_sync_mode sync_mode, void *owner, 1366 bool intr) 1367 { 1368 struct amdgpu_sync sync; 1369 int r; 1370 1371 amdgpu_sync_create(&sync); 1372 amdgpu_sync_resv(adev, &sync, resv, sync_mode, owner); 1373 r = amdgpu_sync_wait(&sync, intr); 1374 amdgpu_sync_free(&sync); 1375 return r; 1376 } 1377 1378 /** 1379 * amdgpu_bo_sync_wait - Wrapper for amdgpu_bo_sync_wait_resv 1380 * @bo: buffer object to wait for 1381 * @owner: fence owner 1382 * @intr: Whether the wait is interruptible 1383 * 1384 * Wrapper to wait for fences in a BO. 1385 * Returns: 1386 * 0 on success, errno otherwise. 1387 */ 1388 int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr) 1389 { 1390 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1391 1392 return amdgpu_bo_sync_wait_resv(adev, bo->tbo.base.resv, 1393 AMDGPU_SYNC_NE_OWNER, owner, intr); 1394 } 1395 1396 /** 1397 * amdgpu_bo_gpu_offset - return GPU offset of bo 1398 * @bo: amdgpu object for which we query the offset 1399 * 1400 * Note: object should either be pinned or reserved when calling this 1401 * function, it might be useful to add check for this for debugging. 1402 * 1403 * Returns: 1404 * current GPU offset of the object. 1405 */ 1406 u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo) 1407 { 1408 WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_SYSTEM); 1409 WARN_ON_ONCE(!dma_resv_is_locked(bo->tbo.base.resv) && 1410 !bo->tbo.pin_count && bo->tbo.type != ttm_bo_type_kernel); 1411 WARN_ON_ONCE(bo->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET); 1412 WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_VRAM && 1413 !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)); 1414 1415 return amdgpu_bo_gpu_offset_no_check(bo); 1416 } 1417 1418 /** 1419 * amdgpu_bo_gpu_offset_no_check - return GPU offset of bo 1420 * @bo: amdgpu object for which we query the offset 1421 * 1422 * Returns: 1423 * current GPU offset of the object without raising warnings. 1424 */ 1425 u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo) 1426 { 1427 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1428 uint64_t offset = AMDGPU_BO_INVALID_OFFSET; 1429 1430 if (bo->tbo.resource->mem_type == TTM_PL_TT) 1431 offset = amdgpu_gmc_agp_addr(&bo->tbo); 1432 1433 if (offset == AMDGPU_BO_INVALID_OFFSET) 1434 offset = (bo->tbo.resource->start << PAGE_SHIFT) + 1435 amdgpu_ttm_domain_start(adev, bo->tbo.resource->mem_type); 1436 1437 return amdgpu_gmc_sign_extend(offset); 1438 } 1439 1440 /** 1441 * amdgpu_bo_get_preferred_domain - get preferred domain 1442 * @adev: amdgpu device object 1443 * @domain: allowed :ref:`memory domains <amdgpu_memory_domains>` 1444 * 1445 * Returns: 1446 * Which of the allowed domains is preferred for allocating the BO. 1447 */ 1448 uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev, 1449 uint32_t domain) 1450 { 1451 if ((domain == (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) && 1452 ((adev->asic_type == CHIP_CARRIZO) || (adev->asic_type == CHIP_STONEY))) { 1453 domain = AMDGPU_GEM_DOMAIN_VRAM; 1454 if (adev->gmc.real_vram_size <= AMDGPU_SG_THRESHOLD) 1455 domain = AMDGPU_GEM_DOMAIN_GTT; 1456 } 1457 return domain; 1458 } 1459 1460 #if defined(CONFIG_DEBUG_FS) 1461 #define amdgpu_bo_print_flag(m, bo, flag) \ 1462 do { \ 1463 if (bo->flags & (AMDGPU_GEM_CREATE_ ## flag)) { \ 1464 seq_printf((m), " " #flag); \ 1465 } \ 1466 } while (0) 1467 1468 /** 1469 * amdgpu_bo_print_info - print BO info in debugfs file 1470 * 1471 * @id: Index or Id of the BO 1472 * @bo: Requested BO for printing info 1473 * @m: debugfs file 1474 * 1475 * Print BO information in debugfs file 1476 * 1477 * Returns: 1478 * Size of the BO in bytes. 1479 */ 1480 u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m) 1481 { 1482 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1483 struct dma_buf_attachment *attachment; 1484 struct dma_buf *dma_buf; 1485 const char *placement; 1486 unsigned int pin_count; 1487 u64 size; 1488 1489 if (dma_resv_trylock(bo->tbo.base.resv)) { 1490 if (!bo->tbo.resource) { 1491 placement = "NONE"; 1492 } else { 1493 switch (bo->tbo.resource->mem_type) { 1494 case TTM_PL_VRAM: 1495 if (amdgpu_res_cpu_visible(adev, bo->tbo.resource)) 1496 placement = "VRAM VISIBLE"; 1497 else 1498 placement = "VRAM"; 1499 break; 1500 case TTM_PL_TT: 1501 placement = "GTT"; 1502 break; 1503 case AMDGPU_PL_GDS: 1504 placement = "GDS"; 1505 break; 1506 case AMDGPU_PL_GWS: 1507 placement = "GWS"; 1508 break; 1509 case AMDGPU_PL_OA: 1510 placement = "OA"; 1511 break; 1512 case AMDGPU_PL_PREEMPT: 1513 placement = "PREEMPTIBLE"; 1514 break; 1515 case AMDGPU_PL_DOORBELL: 1516 placement = "DOORBELL"; 1517 break; 1518 case TTM_PL_SYSTEM: 1519 default: 1520 placement = "CPU"; 1521 break; 1522 } 1523 } 1524 dma_resv_unlock(bo->tbo.base.resv); 1525 } else { 1526 placement = "UNKNOWN"; 1527 } 1528 1529 size = amdgpu_bo_size(bo); 1530 seq_printf(m, "\t\t0x%08x: %12lld byte %s", 1531 id, size, placement); 1532 1533 pin_count = READ_ONCE(bo->tbo.pin_count); 1534 if (pin_count) 1535 seq_printf(m, " pin count %d", pin_count); 1536 1537 dma_buf = READ_ONCE(bo->tbo.base.dma_buf); 1538 attachment = READ_ONCE(bo->tbo.base.import_attach); 1539 1540 if (attachment) 1541 seq_printf(m, " imported from ino:%lu", file_inode(dma_buf->file)->i_ino); 1542 else if (dma_buf) 1543 seq_printf(m, " exported as ino:%lu", file_inode(dma_buf->file)->i_ino); 1544 1545 amdgpu_bo_print_flag(m, bo, CPU_ACCESS_REQUIRED); 1546 amdgpu_bo_print_flag(m, bo, NO_CPU_ACCESS); 1547 amdgpu_bo_print_flag(m, bo, CPU_GTT_USWC); 1548 amdgpu_bo_print_flag(m, bo, VRAM_CLEARED); 1549 amdgpu_bo_print_flag(m, bo, VRAM_CONTIGUOUS); 1550 amdgpu_bo_print_flag(m, bo, VM_ALWAYS_VALID); 1551 amdgpu_bo_print_flag(m, bo, EXPLICIT_SYNC); 1552 1553 seq_puts(m, "\n"); 1554 1555 return size; 1556 } 1557 #endif 1558