1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * NVIDIA Tegra DRM GEM helper functions 4 * 5 * Copyright (C) 2012 Sascha Hauer, Pengutronix 6 * Copyright (C) 2013-2015 NVIDIA CORPORATION, All rights reserved. 7 * 8 * Based on the GEM/CMA helpers 9 * 10 * Copyright (c) 2011 Samsung Electronics Co., Ltd. 11 */ 12 13 #include <linux/dma-buf.h> 14 #include <linux/iommu.h> 15 16 #include <drm/drm_drv.h> 17 #include <drm/drm_prime.h> 18 #include <drm/tegra_drm.h> 19 20 #include "drm.h" 21 #include "gem.h" 22 23 static void tegra_bo_put(struct host1x_bo *bo) 24 { 25 struct tegra_bo *obj = host1x_to_tegra_bo(bo); 26 27 drm_gem_object_put_unlocked(&obj->gem); 28 } 29 30 /* XXX move this into lib/scatterlist.c? */ 31 static int sg_alloc_table_from_sg(struct sg_table *sgt, struct scatterlist *sg, 32 unsigned int nents, gfp_t gfp_mask) 33 { 34 struct scatterlist *dst; 35 unsigned int i; 36 int err; 37 38 err = sg_alloc_table(sgt, nents, gfp_mask); 39 if (err < 0) 40 return err; 41 42 dst = sgt->sgl; 43 44 for (i = 0; i < nents; i++) { 45 sg_set_page(dst, sg_page(sg), sg->length, 0); 46 dst = sg_next(dst); 47 sg = sg_next(sg); 48 } 49 50 return 0; 51 } 52 53 static struct sg_table *tegra_bo_pin(struct device *dev, struct host1x_bo *bo, 54 dma_addr_t *phys) 55 { 56 struct tegra_bo *obj = host1x_to_tegra_bo(bo); 57 struct sg_table *sgt; 58 int err; 59 60 /* 61 * If we've manually mapped the buffer object through the IOMMU, make 62 * sure to return the IOVA address of our mapping. 63 * 64 * Similarly, for buffers that have been allocated by the DMA API the 65 * physical address can be used for devices that are not attached to 66 * an IOMMU. For these devices, callers must pass a valid pointer via 67 * the @phys argument. 68 * 69 * Imported buffers were also already mapped at import time, so the 70 * existing mapping can be reused. 71 */ 72 if (phys) { 73 *phys = obj->iova; 74 return NULL; 75 } 76 77 /* 78 * If we don't have a mapping for this buffer yet, return an SG table 79 * so that host1x can do the mapping for us via the DMA API. 80 */ 81 sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); 82 if (!sgt) 83 return ERR_PTR(-ENOMEM); 84 85 if (obj->pages) { 86 /* 87 * If the buffer object was allocated from the explicit IOMMU 88 * API code paths, construct an SG table from the pages. 89 */ 90 err = sg_alloc_table_from_pages(sgt, obj->pages, obj->num_pages, 91 0, obj->gem.size, GFP_KERNEL); 92 if (err < 0) 93 goto free; 94 } else if (obj->sgt) { 95 /* 96 * If the buffer object already has an SG table but no pages 97 * were allocated for it, it means the buffer was imported and 98 * the SG table needs to be copied to avoid overwriting any 99 * other potential users of the original SG table. 100 */ 101 err = sg_alloc_table_from_sg(sgt, obj->sgt->sgl, obj->sgt->nents, 102 GFP_KERNEL); 103 if (err < 0) 104 goto free; 105 } else { 106 /* 107 * If the buffer object had no pages allocated and if it was 108 * not imported, it had to be allocated with the DMA API, so 109 * the DMA API helper can be used. 110 */ 111 err = dma_get_sgtable(dev, sgt, obj->vaddr, obj->iova, 112 obj->gem.size); 113 if (err < 0) 114 goto free; 115 } 116 117 return sgt; 118 119 free: 120 kfree(sgt); 121 return ERR_PTR(err); 122 } 123 124 static void tegra_bo_unpin(struct device *dev, struct sg_table *sgt) 125 { 126 if (sgt) { 127 sg_free_table(sgt); 128 kfree(sgt); 129 } 130 } 131 132 static void *tegra_bo_mmap(struct host1x_bo *bo) 133 { 134 struct tegra_bo *obj = host1x_to_tegra_bo(bo); 135 136 if (obj->vaddr) 137 return obj->vaddr; 138 else if (obj->gem.import_attach) 139 return dma_buf_vmap(obj->gem.import_attach->dmabuf); 140 else 141 return vmap(obj->pages, obj->num_pages, VM_MAP, 142 pgprot_writecombine(PAGE_KERNEL)); 143 } 144 145 static void tegra_bo_munmap(struct host1x_bo *bo, void *addr) 146 { 147 struct tegra_bo *obj = host1x_to_tegra_bo(bo); 148 149 if (obj->vaddr) 150 return; 151 else if (obj->gem.import_attach) 152 dma_buf_vunmap(obj->gem.import_attach->dmabuf, addr); 153 else 154 vunmap(addr); 155 } 156 157 static void *tegra_bo_kmap(struct host1x_bo *bo, unsigned int page) 158 { 159 struct tegra_bo *obj = host1x_to_tegra_bo(bo); 160 161 if (obj->vaddr) 162 return obj->vaddr + page * PAGE_SIZE; 163 else if (obj->gem.import_attach) 164 return dma_buf_kmap(obj->gem.import_attach->dmabuf, page); 165 else 166 return vmap(obj->pages + page, 1, VM_MAP, 167 pgprot_writecombine(PAGE_KERNEL)); 168 } 169 170 static void tegra_bo_kunmap(struct host1x_bo *bo, unsigned int page, 171 void *addr) 172 { 173 struct tegra_bo *obj = host1x_to_tegra_bo(bo); 174 175 if (obj->vaddr) 176 return; 177 else if (obj->gem.import_attach) 178 dma_buf_kunmap(obj->gem.import_attach->dmabuf, page, addr); 179 else 180 vunmap(addr); 181 } 182 183 static struct host1x_bo *tegra_bo_get(struct host1x_bo *bo) 184 { 185 struct tegra_bo *obj = host1x_to_tegra_bo(bo); 186 187 drm_gem_object_get(&obj->gem); 188 189 return bo; 190 } 191 192 static const struct host1x_bo_ops tegra_bo_ops = { 193 .get = tegra_bo_get, 194 .put = tegra_bo_put, 195 .pin = tegra_bo_pin, 196 .unpin = tegra_bo_unpin, 197 .mmap = tegra_bo_mmap, 198 .munmap = tegra_bo_munmap, 199 .kmap = tegra_bo_kmap, 200 .kunmap = tegra_bo_kunmap, 201 }; 202 203 static int tegra_bo_iommu_map(struct tegra_drm *tegra, struct tegra_bo *bo) 204 { 205 int prot = IOMMU_READ | IOMMU_WRITE; 206 int err; 207 208 if (bo->mm) 209 return -EBUSY; 210 211 bo->mm = kzalloc(sizeof(*bo->mm), GFP_KERNEL); 212 if (!bo->mm) 213 return -ENOMEM; 214 215 mutex_lock(&tegra->mm_lock); 216 217 err = drm_mm_insert_node_generic(&tegra->mm, 218 bo->mm, bo->gem.size, PAGE_SIZE, 0, 0); 219 if (err < 0) { 220 dev_err(tegra->drm->dev, "out of I/O virtual memory: %d\n", 221 err); 222 goto unlock; 223 } 224 225 bo->iova = bo->mm->start; 226 227 bo->size = iommu_map_sg(tegra->domain, bo->iova, bo->sgt->sgl, 228 bo->sgt->nents, prot); 229 if (!bo->size) { 230 dev_err(tegra->drm->dev, "failed to map buffer\n"); 231 err = -ENOMEM; 232 goto remove; 233 } 234 235 mutex_unlock(&tegra->mm_lock); 236 237 return 0; 238 239 remove: 240 drm_mm_remove_node(bo->mm); 241 unlock: 242 mutex_unlock(&tegra->mm_lock); 243 kfree(bo->mm); 244 return err; 245 } 246 247 static int tegra_bo_iommu_unmap(struct tegra_drm *tegra, struct tegra_bo *bo) 248 { 249 if (!bo->mm) 250 return 0; 251 252 mutex_lock(&tegra->mm_lock); 253 iommu_unmap(tegra->domain, bo->iova, bo->size); 254 drm_mm_remove_node(bo->mm); 255 mutex_unlock(&tegra->mm_lock); 256 257 kfree(bo->mm); 258 259 return 0; 260 } 261 262 static struct tegra_bo *tegra_bo_alloc_object(struct drm_device *drm, 263 size_t size) 264 { 265 struct tegra_bo *bo; 266 int err; 267 268 bo = kzalloc(sizeof(*bo), GFP_KERNEL); 269 if (!bo) 270 return ERR_PTR(-ENOMEM); 271 272 host1x_bo_init(&bo->base, &tegra_bo_ops); 273 size = round_up(size, PAGE_SIZE); 274 275 err = drm_gem_object_init(drm, &bo->gem, size); 276 if (err < 0) 277 goto free; 278 279 err = drm_gem_create_mmap_offset(&bo->gem); 280 if (err < 0) 281 goto release; 282 283 return bo; 284 285 release: 286 drm_gem_object_release(&bo->gem); 287 free: 288 kfree(bo); 289 return ERR_PTR(err); 290 } 291 292 static void tegra_bo_free(struct drm_device *drm, struct tegra_bo *bo) 293 { 294 if (bo->pages) { 295 dma_unmap_sg(drm->dev, bo->sgt->sgl, bo->sgt->nents, 296 DMA_FROM_DEVICE); 297 drm_gem_put_pages(&bo->gem, bo->pages, true, true); 298 sg_free_table(bo->sgt); 299 kfree(bo->sgt); 300 } else if (bo->vaddr) { 301 dma_free_wc(drm->dev, bo->gem.size, bo->vaddr, bo->iova); 302 } 303 } 304 305 static int tegra_bo_get_pages(struct drm_device *drm, struct tegra_bo *bo) 306 { 307 int err; 308 309 bo->pages = drm_gem_get_pages(&bo->gem); 310 if (IS_ERR(bo->pages)) 311 return PTR_ERR(bo->pages); 312 313 bo->num_pages = bo->gem.size >> PAGE_SHIFT; 314 315 bo->sgt = drm_prime_pages_to_sg(bo->pages, bo->num_pages); 316 if (IS_ERR(bo->sgt)) { 317 err = PTR_ERR(bo->sgt); 318 goto put_pages; 319 } 320 321 err = dma_map_sg(drm->dev, bo->sgt->sgl, bo->sgt->nents, 322 DMA_FROM_DEVICE); 323 if (err == 0) { 324 err = -EFAULT; 325 goto free_sgt; 326 } 327 328 return 0; 329 330 free_sgt: 331 sg_free_table(bo->sgt); 332 kfree(bo->sgt); 333 put_pages: 334 drm_gem_put_pages(&bo->gem, bo->pages, false, false); 335 return err; 336 } 337 338 static int tegra_bo_alloc(struct drm_device *drm, struct tegra_bo *bo) 339 { 340 struct tegra_drm *tegra = drm->dev_private; 341 int err; 342 343 if (tegra->domain) { 344 err = tegra_bo_get_pages(drm, bo); 345 if (err < 0) 346 return err; 347 348 err = tegra_bo_iommu_map(tegra, bo); 349 if (err < 0) { 350 tegra_bo_free(drm, bo); 351 return err; 352 } 353 } else { 354 size_t size = bo->gem.size; 355 356 bo->vaddr = dma_alloc_wc(drm->dev, size, &bo->iova, 357 GFP_KERNEL | __GFP_NOWARN); 358 if (!bo->vaddr) { 359 dev_err(drm->dev, 360 "failed to allocate buffer of size %zu\n", 361 size); 362 return -ENOMEM; 363 } 364 } 365 366 return 0; 367 } 368 369 struct tegra_bo *tegra_bo_create(struct drm_device *drm, size_t size, 370 unsigned long flags) 371 { 372 struct tegra_bo *bo; 373 int err; 374 375 bo = tegra_bo_alloc_object(drm, size); 376 if (IS_ERR(bo)) 377 return bo; 378 379 err = tegra_bo_alloc(drm, bo); 380 if (err < 0) 381 goto release; 382 383 if (flags & DRM_TEGRA_GEM_CREATE_TILED) 384 bo->tiling.mode = TEGRA_BO_TILING_MODE_TILED; 385 386 if (flags & DRM_TEGRA_GEM_CREATE_BOTTOM_UP) 387 bo->flags |= TEGRA_BO_BOTTOM_UP; 388 389 return bo; 390 391 release: 392 drm_gem_object_release(&bo->gem); 393 kfree(bo); 394 return ERR_PTR(err); 395 } 396 397 struct tegra_bo *tegra_bo_create_with_handle(struct drm_file *file, 398 struct drm_device *drm, 399 size_t size, 400 unsigned long flags, 401 u32 *handle) 402 { 403 struct tegra_bo *bo; 404 int err; 405 406 bo = tegra_bo_create(drm, size, flags); 407 if (IS_ERR(bo)) 408 return bo; 409 410 err = drm_gem_handle_create(file, &bo->gem, handle); 411 if (err) { 412 tegra_bo_free_object(&bo->gem); 413 return ERR_PTR(err); 414 } 415 416 drm_gem_object_put_unlocked(&bo->gem); 417 418 return bo; 419 } 420 421 static struct tegra_bo *tegra_bo_import(struct drm_device *drm, 422 struct dma_buf *buf) 423 { 424 struct tegra_drm *tegra = drm->dev_private; 425 struct dma_buf_attachment *attach; 426 struct tegra_bo *bo; 427 int err; 428 429 bo = tegra_bo_alloc_object(drm, buf->size); 430 if (IS_ERR(bo)) 431 return bo; 432 433 attach = dma_buf_attach(buf, drm->dev); 434 if (IS_ERR(attach)) { 435 err = PTR_ERR(attach); 436 goto free; 437 } 438 439 get_dma_buf(buf); 440 441 bo->sgt = dma_buf_map_attachment(attach, DMA_TO_DEVICE); 442 if (IS_ERR(bo->sgt)) { 443 err = PTR_ERR(bo->sgt); 444 goto detach; 445 } 446 447 if (tegra->domain) { 448 err = tegra_bo_iommu_map(tegra, bo); 449 if (err < 0) 450 goto detach; 451 } 452 453 bo->gem.import_attach = attach; 454 455 return bo; 456 457 detach: 458 if (!IS_ERR_OR_NULL(bo->sgt)) 459 dma_buf_unmap_attachment(attach, bo->sgt, DMA_TO_DEVICE); 460 461 dma_buf_detach(buf, attach); 462 dma_buf_put(buf); 463 free: 464 drm_gem_object_release(&bo->gem); 465 kfree(bo); 466 return ERR_PTR(err); 467 } 468 469 void tegra_bo_free_object(struct drm_gem_object *gem) 470 { 471 struct tegra_drm *tegra = gem->dev->dev_private; 472 struct tegra_bo *bo = to_tegra_bo(gem); 473 474 if (tegra->domain) 475 tegra_bo_iommu_unmap(tegra, bo); 476 477 if (gem->import_attach) { 478 dma_buf_unmap_attachment(gem->import_attach, bo->sgt, 479 DMA_TO_DEVICE); 480 drm_prime_gem_destroy(gem, NULL); 481 } else { 482 tegra_bo_free(gem->dev, bo); 483 } 484 485 drm_gem_object_release(gem); 486 kfree(bo); 487 } 488 489 int tegra_bo_dumb_create(struct drm_file *file, struct drm_device *drm, 490 struct drm_mode_create_dumb *args) 491 { 492 unsigned int min_pitch = DIV_ROUND_UP(args->width * args->bpp, 8); 493 struct tegra_drm *tegra = drm->dev_private; 494 struct tegra_bo *bo; 495 496 args->pitch = round_up(min_pitch, tegra->pitch_align); 497 args->size = args->pitch * args->height; 498 499 bo = tegra_bo_create_with_handle(file, drm, args->size, 0, 500 &args->handle); 501 if (IS_ERR(bo)) 502 return PTR_ERR(bo); 503 504 return 0; 505 } 506 507 static vm_fault_t tegra_bo_fault(struct vm_fault *vmf) 508 { 509 struct vm_area_struct *vma = vmf->vma; 510 struct drm_gem_object *gem = vma->vm_private_data; 511 struct tegra_bo *bo = to_tegra_bo(gem); 512 struct page *page; 513 pgoff_t offset; 514 515 if (!bo->pages) 516 return VM_FAULT_SIGBUS; 517 518 offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT; 519 page = bo->pages[offset]; 520 521 return vmf_insert_page(vma, vmf->address, page); 522 } 523 524 const struct vm_operations_struct tegra_bo_vm_ops = { 525 .fault = tegra_bo_fault, 526 .open = drm_gem_vm_open, 527 .close = drm_gem_vm_close, 528 }; 529 530 int __tegra_gem_mmap(struct drm_gem_object *gem, struct vm_area_struct *vma) 531 { 532 struct tegra_bo *bo = to_tegra_bo(gem); 533 534 if (!bo->pages) { 535 unsigned long vm_pgoff = vma->vm_pgoff; 536 int err; 537 538 /* 539 * Clear the VM_PFNMAP flag that was set by drm_gem_mmap(), 540 * and set the vm_pgoff (used as a fake buffer offset by DRM) 541 * to 0 as we want to map the whole buffer. 542 */ 543 vma->vm_flags &= ~VM_PFNMAP; 544 vma->vm_pgoff = 0; 545 546 err = dma_mmap_wc(gem->dev->dev, vma, bo->vaddr, bo->iova, 547 gem->size); 548 if (err < 0) { 549 drm_gem_vm_close(vma); 550 return err; 551 } 552 553 vma->vm_pgoff = vm_pgoff; 554 } else { 555 pgprot_t prot = vm_get_page_prot(vma->vm_flags); 556 557 vma->vm_flags |= VM_MIXEDMAP; 558 vma->vm_flags &= ~VM_PFNMAP; 559 560 vma->vm_page_prot = pgprot_writecombine(prot); 561 } 562 563 return 0; 564 } 565 566 int tegra_drm_mmap(struct file *file, struct vm_area_struct *vma) 567 { 568 struct drm_gem_object *gem; 569 int err; 570 571 err = drm_gem_mmap(file, vma); 572 if (err < 0) 573 return err; 574 575 gem = vma->vm_private_data; 576 577 return __tegra_gem_mmap(gem, vma); 578 } 579 580 static struct sg_table * 581 tegra_gem_prime_map_dma_buf(struct dma_buf_attachment *attach, 582 enum dma_data_direction dir) 583 { 584 struct drm_gem_object *gem = attach->dmabuf->priv; 585 struct tegra_bo *bo = to_tegra_bo(gem); 586 struct sg_table *sgt; 587 588 sgt = kmalloc(sizeof(*sgt), GFP_KERNEL); 589 if (!sgt) 590 return NULL; 591 592 if (bo->pages) { 593 if (sg_alloc_table_from_pages(sgt, bo->pages, bo->num_pages, 594 0, gem->size, GFP_KERNEL) < 0) 595 goto free; 596 } else { 597 if (dma_get_sgtable(attach->dev, sgt, bo->vaddr, bo->iova, 598 gem->size) < 0) 599 goto free; 600 } 601 602 if (dma_map_sg(attach->dev, sgt->sgl, sgt->nents, dir) == 0) 603 goto free; 604 605 return sgt; 606 607 free: 608 sg_free_table(sgt); 609 kfree(sgt); 610 return NULL; 611 } 612 613 static void tegra_gem_prime_unmap_dma_buf(struct dma_buf_attachment *attach, 614 struct sg_table *sgt, 615 enum dma_data_direction dir) 616 { 617 struct drm_gem_object *gem = attach->dmabuf->priv; 618 struct tegra_bo *bo = to_tegra_bo(gem); 619 620 if (bo->pages) 621 dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents, dir); 622 623 sg_free_table(sgt); 624 kfree(sgt); 625 } 626 627 static void tegra_gem_prime_release(struct dma_buf *buf) 628 { 629 drm_gem_dmabuf_release(buf); 630 } 631 632 static int tegra_gem_prime_begin_cpu_access(struct dma_buf *buf, 633 enum dma_data_direction direction) 634 { 635 struct drm_gem_object *gem = buf->priv; 636 struct tegra_bo *bo = to_tegra_bo(gem); 637 struct drm_device *drm = gem->dev; 638 639 if (bo->pages) 640 dma_sync_sg_for_cpu(drm->dev, bo->sgt->sgl, bo->sgt->nents, 641 DMA_FROM_DEVICE); 642 643 return 0; 644 } 645 646 static int tegra_gem_prime_end_cpu_access(struct dma_buf *buf, 647 enum dma_data_direction direction) 648 { 649 struct drm_gem_object *gem = buf->priv; 650 struct tegra_bo *bo = to_tegra_bo(gem); 651 struct drm_device *drm = gem->dev; 652 653 if (bo->pages) 654 dma_sync_sg_for_device(drm->dev, bo->sgt->sgl, bo->sgt->nents, 655 DMA_TO_DEVICE); 656 657 return 0; 658 } 659 660 static void *tegra_gem_prime_kmap(struct dma_buf *buf, unsigned long page) 661 { 662 return NULL; 663 } 664 665 static void tegra_gem_prime_kunmap(struct dma_buf *buf, unsigned long page, 666 void *addr) 667 { 668 } 669 670 static int tegra_gem_prime_mmap(struct dma_buf *buf, struct vm_area_struct *vma) 671 { 672 struct drm_gem_object *gem = buf->priv; 673 int err; 674 675 err = drm_gem_mmap_obj(gem, gem->size, vma); 676 if (err < 0) 677 return err; 678 679 return __tegra_gem_mmap(gem, vma); 680 } 681 682 static void *tegra_gem_prime_vmap(struct dma_buf *buf) 683 { 684 struct drm_gem_object *gem = buf->priv; 685 struct tegra_bo *bo = to_tegra_bo(gem); 686 687 return bo->vaddr; 688 } 689 690 static void tegra_gem_prime_vunmap(struct dma_buf *buf, void *vaddr) 691 { 692 } 693 694 static const struct dma_buf_ops tegra_gem_prime_dmabuf_ops = { 695 .map_dma_buf = tegra_gem_prime_map_dma_buf, 696 .unmap_dma_buf = tegra_gem_prime_unmap_dma_buf, 697 .release = tegra_gem_prime_release, 698 .begin_cpu_access = tegra_gem_prime_begin_cpu_access, 699 .end_cpu_access = tegra_gem_prime_end_cpu_access, 700 .map = tegra_gem_prime_kmap, 701 .unmap = tegra_gem_prime_kunmap, 702 .mmap = tegra_gem_prime_mmap, 703 .vmap = tegra_gem_prime_vmap, 704 .vunmap = tegra_gem_prime_vunmap, 705 }; 706 707 struct dma_buf *tegra_gem_prime_export(struct drm_gem_object *gem, 708 int flags) 709 { 710 DEFINE_DMA_BUF_EXPORT_INFO(exp_info); 711 712 exp_info.exp_name = KBUILD_MODNAME; 713 exp_info.owner = gem->dev->driver->fops->owner; 714 exp_info.ops = &tegra_gem_prime_dmabuf_ops; 715 exp_info.size = gem->size; 716 exp_info.flags = flags; 717 exp_info.priv = gem; 718 719 return drm_gem_dmabuf_export(gem->dev, &exp_info); 720 } 721 722 struct drm_gem_object *tegra_gem_prime_import(struct drm_device *drm, 723 struct dma_buf *buf) 724 { 725 struct tegra_bo *bo; 726 727 if (buf->ops == &tegra_gem_prime_dmabuf_ops) { 728 struct drm_gem_object *gem = buf->priv; 729 730 if (gem->dev == drm) { 731 drm_gem_object_get(gem); 732 return gem; 733 } 734 } 735 736 bo = tegra_bo_import(drm, buf); 737 if (IS_ERR(bo)) 738 return ERR_CAST(bo); 739 740 return &bo->gem; 741 } 742