1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Copyright (C) 2014-2018 Broadcom 4 * Copyright (C) 2023 Raspberry Pi 5 */ 6 7 #include <drm/drm_syncobj.h> 8 9 #include "v3d_drv.h" 10 #include "v3d_regs.h" 11 #include "v3d_trace.h" 12 13 /* Takes the reservation lock on all the BOs being referenced, so that 14 * at queue submit time we can update the reservations. 15 * 16 * We don't lock the RCL the tile alloc/state BOs, or overflow memory 17 * (all of which are on exec->unref_list). They're entirely private 18 * to v3d, so we don't attach dma-buf fences to them. 19 */ 20 static int 21 v3d_lock_bo_reservations(struct v3d_job *job, 22 struct ww_acquire_ctx *acquire_ctx) 23 { 24 int i, ret; 25 26 ret = drm_gem_lock_reservations(job->bo, job->bo_count, acquire_ctx); 27 if (ret) 28 return ret; 29 30 for (i = 0; i < job->bo_count; i++) { 31 ret = dma_resv_reserve_fences(job->bo[i]->resv, 1); 32 if (ret) 33 goto fail; 34 35 ret = drm_sched_job_add_implicit_dependencies(&job->base, 36 job->bo[i], true); 37 if (ret) 38 goto fail; 39 } 40 41 return 0; 42 43 fail: 44 drm_gem_unlock_reservations(job->bo, job->bo_count, acquire_ctx); 45 return ret; 46 } 47 48 /** 49 * v3d_lookup_bos() - Sets up job->bo[] with the GEM objects 50 * referenced by the job. 51 * @dev: DRM device 52 * @file_priv: DRM file for this fd 53 * @job: V3D job being set up 54 * @bo_handles: GEM handles 55 * @bo_count: Number of GEM handles passed in 56 * 57 * The command validator needs to reference BOs by their index within 58 * the submitted job's BO list. This does the validation of the job's 59 * BO list and reference counting for the lifetime of the job. 60 * 61 * Note that this function doesn't need to unreference the BOs on 62 * failure, because that will happen at v3d_exec_cleanup() time. 63 */ 64 static int 65 v3d_lookup_bos(struct drm_device *dev, 66 struct drm_file *file_priv, 67 struct v3d_job *job, 68 u64 bo_handles, 69 u32 bo_count) 70 { 71 job->bo_count = bo_count; 72 73 if (!job->bo_count) { 74 /* See comment on bo_index for why we have to check 75 * this. 76 */ 77 DRM_DEBUG("Rendering requires BOs\n"); 78 return -EINVAL; 79 } 80 81 return drm_gem_objects_lookup(file_priv, 82 (void __user *)(uintptr_t)bo_handles, 83 job->bo_count, &job->bo); 84 } 85 86 static void 87 v3d_job_free(struct kref *ref) 88 { 89 struct v3d_job *job = container_of(ref, struct v3d_job, refcount); 90 int i; 91 92 if (job->bo) { 93 for (i = 0; i < job->bo_count; i++) 94 drm_gem_object_put(job->bo[i]); 95 kvfree(job->bo); 96 } 97 98 dma_fence_put(job->irq_fence); 99 dma_fence_put(job->done_fence); 100 101 if (job->perfmon) 102 v3d_perfmon_put(job->perfmon); 103 104 kfree(job); 105 } 106 107 static void 108 v3d_render_job_free(struct kref *ref) 109 { 110 struct v3d_render_job *job = container_of(ref, struct v3d_render_job, 111 base.refcount); 112 struct v3d_bo *bo, *save; 113 114 list_for_each_entry_safe(bo, save, &job->unref_list, unref_head) { 115 drm_gem_object_put(&bo->base.base); 116 } 117 118 v3d_job_free(ref); 119 } 120 121 void v3d_job_cleanup(struct v3d_job *job) 122 { 123 if (!job) 124 return; 125 126 drm_sched_job_cleanup(&job->base); 127 v3d_job_put(job); 128 } 129 130 void v3d_job_put(struct v3d_job *job) 131 { 132 if (!job) 133 return; 134 135 kref_put(&job->refcount, job->free); 136 } 137 138 static int 139 v3d_job_allocate(void **container, size_t size) 140 { 141 *container = kcalloc(1, size, GFP_KERNEL); 142 if (!*container) { 143 DRM_ERROR("Cannot allocate memory for V3D job.\n"); 144 return -ENOMEM; 145 } 146 147 return 0; 148 } 149 150 static void 151 v3d_job_deallocate(void **container) 152 { 153 kfree(*container); 154 *container = NULL; 155 } 156 157 static int 158 v3d_job_init(struct v3d_dev *v3d, struct drm_file *file_priv, 159 struct v3d_job *job, void (*free)(struct kref *ref), 160 u32 in_sync, struct v3d_submit_ext *se, enum v3d_queue queue) 161 { 162 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 163 bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC); 164 int ret, i; 165 166 job->v3d = v3d; 167 job->free = free; 168 job->file = file_priv; 169 170 ret = drm_sched_job_init(&job->base, &v3d_priv->sched_entity[queue], 171 1, v3d_priv); 172 if (ret) 173 return ret; 174 175 if (has_multisync) { 176 if (se->in_sync_count && se->wait_stage == queue) { 177 struct drm_v3d_sem __user *handle = u64_to_user_ptr(se->in_syncs); 178 179 for (i = 0; i < se->in_sync_count; i++) { 180 struct drm_v3d_sem in; 181 182 if (copy_from_user(&in, handle++, sizeof(in))) { 183 ret = -EFAULT; 184 DRM_DEBUG("Failed to copy wait dep handle.\n"); 185 goto fail_deps; 186 } 187 ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, in.handle, 0); 188 189 // TODO: Investigate why this was filtered out for the IOCTL. 190 if (ret && ret != -ENOENT) 191 goto fail_deps; 192 } 193 } 194 } else { 195 ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, in_sync, 0); 196 197 // TODO: Investigate why this was filtered out for the IOCTL. 198 if (ret && ret != -ENOENT) 199 goto fail_deps; 200 } 201 202 kref_init(&job->refcount); 203 204 return 0; 205 206 fail_deps: 207 drm_sched_job_cleanup(&job->base); 208 return ret; 209 } 210 211 static void 212 v3d_push_job(struct v3d_job *job) 213 { 214 drm_sched_job_arm(&job->base); 215 216 job->done_fence = dma_fence_get(&job->base.s_fence->finished); 217 218 /* put by scheduler job completion */ 219 kref_get(&job->refcount); 220 221 drm_sched_entity_push_job(&job->base); 222 } 223 224 static void 225 v3d_attach_fences_and_unlock_reservation(struct drm_file *file_priv, 226 struct v3d_job *job, 227 struct ww_acquire_ctx *acquire_ctx, 228 u32 out_sync, 229 struct v3d_submit_ext *se, 230 struct dma_fence *done_fence) 231 { 232 struct drm_syncobj *sync_out; 233 bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC); 234 int i; 235 236 for (i = 0; i < job->bo_count; i++) { 237 /* XXX: Use shared fences for read-only objects. */ 238 dma_resv_add_fence(job->bo[i]->resv, job->done_fence, 239 DMA_RESV_USAGE_WRITE); 240 } 241 242 drm_gem_unlock_reservations(job->bo, job->bo_count, acquire_ctx); 243 244 /* Update the return sync object for the job */ 245 /* If it only supports a single signal semaphore*/ 246 if (!has_multisync) { 247 sync_out = drm_syncobj_find(file_priv, out_sync); 248 if (sync_out) { 249 drm_syncobj_replace_fence(sync_out, done_fence); 250 drm_syncobj_put(sync_out); 251 } 252 return; 253 } 254 255 /* If multiple semaphores extension is supported */ 256 if (se->out_sync_count) { 257 for (i = 0; i < se->out_sync_count; i++) { 258 drm_syncobj_replace_fence(se->out_syncs[i].syncobj, 259 done_fence); 260 drm_syncobj_put(se->out_syncs[i].syncobj); 261 } 262 kvfree(se->out_syncs); 263 } 264 } 265 266 static int 267 v3d_setup_csd_jobs_and_bos(struct drm_file *file_priv, 268 struct v3d_dev *v3d, 269 struct drm_v3d_submit_csd *args, 270 struct v3d_csd_job **job, 271 struct v3d_job **clean_job, 272 struct v3d_submit_ext *se, 273 struct ww_acquire_ctx *acquire_ctx) 274 { 275 int ret; 276 277 ret = v3d_job_allocate((void *)job, sizeof(**job)); 278 if (ret) 279 return ret; 280 281 ret = v3d_job_init(v3d, file_priv, &(*job)->base, 282 v3d_job_free, args->in_sync, se, V3D_CSD); 283 if (ret) { 284 v3d_job_deallocate((void *)job); 285 return ret; 286 } 287 288 ret = v3d_job_allocate((void *)clean_job, sizeof(**clean_job)); 289 if (ret) 290 return ret; 291 292 ret = v3d_job_init(v3d, file_priv, *clean_job, 293 v3d_job_free, 0, NULL, V3D_CACHE_CLEAN); 294 if (ret) { 295 v3d_job_deallocate((void *)clean_job); 296 return ret; 297 } 298 299 (*job)->args = *args; 300 301 ret = v3d_lookup_bos(&v3d->drm, file_priv, *clean_job, 302 args->bo_handles, args->bo_handle_count); 303 if (ret) 304 return ret; 305 306 return v3d_lock_bo_reservations(*clean_job, acquire_ctx); 307 } 308 309 static void 310 v3d_put_multisync_post_deps(struct v3d_submit_ext *se) 311 { 312 unsigned int i; 313 314 if (!(se && se->out_sync_count)) 315 return; 316 317 for (i = 0; i < se->out_sync_count; i++) 318 drm_syncobj_put(se->out_syncs[i].syncobj); 319 kvfree(se->out_syncs); 320 } 321 322 static int 323 v3d_get_multisync_post_deps(struct drm_file *file_priv, 324 struct v3d_submit_ext *se, 325 u32 count, u64 handles) 326 { 327 struct drm_v3d_sem __user *post_deps; 328 int i, ret; 329 330 if (!count) 331 return 0; 332 333 se->out_syncs = (struct v3d_submit_outsync *) 334 kvmalloc_array(count, 335 sizeof(struct v3d_submit_outsync), 336 GFP_KERNEL); 337 if (!se->out_syncs) 338 return -ENOMEM; 339 340 post_deps = u64_to_user_ptr(handles); 341 342 for (i = 0; i < count; i++) { 343 struct drm_v3d_sem out; 344 345 if (copy_from_user(&out, post_deps++, sizeof(out))) { 346 ret = -EFAULT; 347 DRM_DEBUG("Failed to copy post dep handles\n"); 348 goto fail; 349 } 350 351 se->out_syncs[i].syncobj = drm_syncobj_find(file_priv, 352 out.handle); 353 if (!se->out_syncs[i].syncobj) { 354 ret = -EINVAL; 355 goto fail; 356 } 357 } 358 se->out_sync_count = count; 359 360 return 0; 361 362 fail: 363 for (i--; i >= 0; i--) 364 drm_syncobj_put(se->out_syncs[i].syncobj); 365 kvfree(se->out_syncs); 366 367 return ret; 368 } 369 370 /* Get data for multiple binary semaphores synchronization. Parse syncobj 371 * to be signaled when job completes (out_sync). 372 */ 373 static int 374 v3d_get_multisync_submit_deps(struct drm_file *file_priv, 375 struct drm_v3d_extension __user *ext, 376 struct v3d_submit_ext *se) 377 { 378 struct drm_v3d_multi_sync multisync; 379 int ret; 380 381 if (se->in_sync_count || se->out_sync_count) { 382 DRM_DEBUG("Two multisync extensions were added to the same job."); 383 return -EINVAL; 384 } 385 386 if (copy_from_user(&multisync, ext, sizeof(multisync))) 387 return -EFAULT; 388 389 if (multisync.pad) 390 return -EINVAL; 391 392 ret = v3d_get_multisync_post_deps(file_priv, se, multisync.out_sync_count, 393 multisync.out_syncs); 394 if (ret) 395 return ret; 396 397 se->in_sync_count = multisync.in_sync_count; 398 se->in_syncs = multisync.in_syncs; 399 se->flags |= DRM_V3D_EXT_ID_MULTI_SYNC; 400 se->wait_stage = multisync.wait_stage; 401 402 return 0; 403 } 404 405 /* Get data for the indirect CSD job submission. */ 406 static int 407 v3d_get_cpu_indirect_csd_params(struct drm_file *file_priv, 408 struct drm_v3d_extension __user *ext, 409 struct v3d_cpu_job *job) 410 { 411 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 412 struct v3d_dev *v3d = v3d_priv->v3d; 413 struct drm_v3d_indirect_csd indirect_csd; 414 struct v3d_indirect_csd_info *info = &job->indirect_csd; 415 416 if (!job) { 417 DRM_DEBUG("CPU job extension was attached to a GPU job.\n"); 418 return -EINVAL; 419 } 420 421 if (job->job_type) { 422 DRM_DEBUG("Two CPU job extensions were added to the same CPU job.\n"); 423 return -EINVAL; 424 } 425 426 if (copy_from_user(&indirect_csd, ext, sizeof(indirect_csd))) 427 return -EFAULT; 428 429 if (!v3d_has_csd(v3d)) { 430 DRM_DEBUG("Attempting CSD submit on non-CSD hardware.\n"); 431 return -EINVAL; 432 } 433 434 job->job_type = V3D_CPU_JOB_TYPE_INDIRECT_CSD; 435 info->offset = indirect_csd.offset; 436 info->wg_size = indirect_csd.wg_size; 437 memcpy(&info->wg_uniform_offsets, &indirect_csd.wg_uniform_offsets, 438 sizeof(indirect_csd.wg_uniform_offsets)); 439 440 info->indirect = drm_gem_object_lookup(file_priv, indirect_csd.indirect); 441 442 return v3d_setup_csd_jobs_and_bos(file_priv, v3d, &indirect_csd.submit, 443 &info->job, &info->clean_job, 444 NULL, &info->acquire_ctx); 445 } 446 447 /* Get data for the query timestamp job submission. */ 448 static int 449 v3d_get_cpu_timestamp_query_params(struct drm_file *file_priv, 450 struct drm_v3d_extension __user *ext, 451 struct v3d_cpu_job *job) 452 { 453 u32 __user *offsets, *syncs; 454 struct drm_v3d_timestamp_query timestamp; 455 unsigned int i; 456 int err; 457 458 if (!job) { 459 DRM_DEBUG("CPU job extension was attached to a GPU job.\n"); 460 return -EINVAL; 461 } 462 463 if (job->job_type) { 464 DRM_DEBUG("Two CPU job extensions were added to the same CPU job.\n"); 465 return -EINVAL; 466 } 467 468 if (copy_from_user(×tamp, ext, sizeof(timestamp))) 469 return -EFAULT; 470 471 if (timestamp.pad) 472 return -EINVAL; 473 474 job->job_type = V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY; 475 476 job->timestamp_query.queries = kvmalloc_array(timestamp.count, 477 sizeof(struct v3d_timestamp_query), 478 GFP_KERNEL); 479 if (!job->timestamp_query.queries) 480 return -ENOMEM; 481 482 offsets = u64_to_user_ptr(timestamp.offsets); 483 syncs = u64_to_user_ptr(timestamp.syncs); 484 485 for (i = 0; i < timestamp.count; i++) { 486 u32 offset, sync; 487 488 if (copy_from_user(&offset, offsets++, sizeof(offset))) { 489 err = -EFAULT; 490 goto error; 491 } 492 493 job->timestamp_query.queries[i].offset = offset; 494 495 if (copy_from_user(&sync, syncs++, sizeof(sync))) { 496 err = -EFAULT; 497 goto error; 498 } 499 500 job->timestamp_query.queries[i].syncobj = drm_syncobj_find(file_priv, sync); 501 if (!job->timestamp_query.queries[i].syncobj) { 502 err = -ENOENT; 503 goto error; 504 } 505 } 506 job->timestamp_query.count = timestamp.count; 507 508 return 0; 509 510 error: 511 v3d_timestamp_query_info_free(&job->timestamp_query, i); 512 return err; 513 } 514 515 static int 516 v3d_get_cpu_reset_timestamp_params(struct drm_file *file_priv, 517 struct drm_v3d_extension __user *ext, 518 struct v3d_cpu_job *job) 519 { 520 u32 __user *syncs; 521 struct drm_v3d_reset_timestamp_query reset; 522 unsigned int i; 523 int err; 524 525 if (!job) { 526 DRM_DEBUG("CPU job extension was attached to a GPU job.\n"); 527 return -EINVAL; 528 } 529 530 if (job->job_type) { 531 DRM_DEBUG("Two CPU job extensions were added to the same CPU job.\n"); 532 return -EINVAL; 533 } 534 535 if (copy_from_user(&reset, ext, sizeof(reset))) 536 return -EFAULT; 537 538 job->job_type = V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY; 539 540 job->timestamp_query.queries = kvmalloc_array(reset.count, 541 sizeof(struct v3d_timestamp_query), 542 GFP_KERNEL); 543 if (!job->timestamp_query.queries) 544 return -ENOMEM; 545 546 syncs = u64_to_user_ptr(reset.syncs); 547 548 for (i = 0; i < reset.count; i++) { 549 u32 sync; 550 551 job->timestamp_query.queries[i].offset = reset.offset + 8 * i; 552 553 if (copy_from_user(&sync, syncs++, sizeof(sync))) { 554 err = -EFAULT; 555 goto error; 556 } 557 558 job->timestamp_query.queries[i].syncobj = drm_syncobj_find(file_priv, sync); 559 if (!job->timestamp_query.queries[i].syncobj) { 560 err = -ENOENT; 561 goto error; 562 } 563 } 564 job->timestamp_query.count = reset.count; 565 566 return 0; 567 568 error: 569 v3d_timestamp_query_info_free(&job->timestamp_query, i); 570 return err; 571 } 572 573 /* Get data for the copy timestamp query results job submission. */ 574 static int 575 v3d_get_cpu_copy_query_results_params(struct drm_file *file_priv, 576 struct drm_v3d_extension __user *ext, 577 struct v3d_cpu_job *job) 578 { 579 u32 __user *offsets, *syncs; 580 struct drm_v3d_copy_timestamp_query copy; 581 unsigned int i; 582 int err; 583 584 if (!job) { 585 DRM_DEBUG("CPU job extension was attached to a GPU job.\n"); 586 return -EINVAL; 587 } 588 589 if (job->job_type) { 590 DRM_DEBUG("Two CPU job extensions were added to the same CPU job.\n"); 591 return -EINVAL; 592 } 593 594 if (copy_from_user(©, ext, sizeof(copy))) 595 return -EFAULT; 596 597 if (copy.pad) 598 return -EINVAL; 599 600 job->job_type = V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY; 601 602 job->timestamp_query.queries = kvmalloc_array(copy.count, 603 sizeof(struct v3d_timestamp_query), 604 GFP_KERNEL); 605 if (!job->timestamp_query.queries) 606 return -ENOMEM; 607 608 offsets = u64_to_user_ptr(copy.offsets); 609 syncs = u64_to_user_ptr(copy.syncs); 610 611 for (i = 0; i < copy.count; i++) { 612 u32 offset, sync; 613 614 if (copy_from_user(&offset, offsets++, sizeof(offset))) { 615 err = -EFAULT; 616 goto error; 617 } 618 619 job->timestamp_query.queries[i].offset = offset; 620 621 if (copy_from_user(&sync, syncs++, sizeof(sync))) { 622 err = -EFAULT; 623 goto error; 624 } 625 626 job->timestamp_query.queries[i].syncobj = drm_syncobj_find(file_priv, sync); 627 if (!job->timestamp_query.queries[i].syncobj) { 628 err = -ENOENT; 629 goto error; 630 } 631 } 632 job->timestamp_query.count = copy.count; 633 634 job->copy.do_64bit = copy.do_64bit; 635 job->copy.do_partial = copy.do_partial; 636 job->copy.availability_bit = copy.availability_bit; 637 job->copy.offset = copy.offset; 638 job->copy.stride = copy.stride; 639 640 return 0; 641 642 error: 643 v3d_timestamp_query_info_free(&job->timestamp_query, i); 644 return err; 645 } 646 647 static int 648 v3d_get_cpu_reset_performance_params(struct drm_file *file_priv, 649 struct drm_v3d_extension __user *ext, 650 struct v3d_cpu_job *job) 651 { 652 u32 __user *syncs; 653 u64 __user *kperfmon_ids; 654 struct drm_v3d_reset_performance_query reset; 655 unsigned int i, j; 656 int err; 657 658 if (!job) { 659 DRM_DEBUG("CPU job extension was attached to a GPU job.\n"); 660 return -EINVAL; 661 } 662 663 if (job->job_type) { 664 DRM_DEBUG("Two CPU job extensions were added to the same CPU job.\n"); 665 return -EINVAL; 666 } 667 668 if (copy_from_user(&reset, ext, sizeof(reset))) 669 return -EFAULT; 670 671 if (reset.nperfmons > V3D_MAX_PERFMONS) 672 return -EINVAL; 673 674 job->job_type = V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY; 675 676 job->performance_query.queries = kvmalloc_array(reset.count, 677 sizeof(struct v3d_performance_query), 678 GFP_KERNEL); 679 if (!job->performance_query.queries) 680 return -ENOMEM; 681 682 syncs = u64_to_user_ptr(reset.syncs); 683 kperfmon_ids = u64_to_user_ptr(reset.kperfmon_ids); 684 685 for (i = 0; i < reset.count; i++) { 686 u32 sync; 687 u64 ids; 688 u32 __user *ids_pointer; 689 u32 id; 690 691 if (copy_from_user(&sync, syncs++, sizeof(sync))) { 692 err = -EFAULT; 693 goto error; 694 } 695 696 if (copy_from_user(&ids, kperfmon_ids++, sizeof(ids))) { 697 err = -EFAULT; 698 goto error; 699 } 700 701 ids_pointer = u64_to_user_ptr(ids); 702 703 for (j = 0; j < reset.nperfmons; j++) { 704 if (copy_from_user(&id, ids_pointer++, sizeof(id))) { 705 err = -EFAULT; 706 goto error; 707 } 708 709 job->performance_query.queries[i].kperfmon_ids[j] = id; 710 } 711 712 job->performance_query.queries[i].syncobj = drm_syncobj_find(file_priv, sync); 713 } 714 job->performance_query.count = reset.count; 715 job->performance_query.nperfmons = reset.nperfmons; 716 717 return 0; 718 719 error: 720 v3d_performance_query_info_free(&job->performance_query, i); 721 return err; 722 } 723 724 static int 725 v3d_get_cpu_copy_performance_query_params(struct drm_file *file_priv, 726 struct drm_v3d_extension __user *ext, 727 struct v3d_cpu_job *job) 728 { 729 u32 __user *syncs; 730 u64 __user *kperfmon_ids; 731 struct drm_v3d_copy_performance_query copy; 732 unsigned int i, j; 733 int err; 734 735 if (!job) { 736 DRM_DEBUG("CPU job extension was attached to a GPU job.\n"); 737 return -EINVAL; 738 } 739 740 if (job->job_type) { 741 DRM_DEBUG("Two CPU job extensions were added to the same CPU job.\n"); 742 return -EINVAL; 743 } 744 745 if (copy_from_user(©, ext, sizeof(copy))) 746 return -EFAULT; 747 748 if (copy.pad) 749 return -EINVAL; 750 751 if (copy.nperfmons > V3D_MAX_PERFMONS) 752 return -EINVAL; 753 754 job->job_type = V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY; 755 756 job->performance_query.queries = kvmalloc_array(copy.count, 757 sizeof(struct v3d_performance_query), 758 GFP_KERNEL); 759 if (!job->performance_query.queries) 760 return -ENOMEM; 761 762 syncs = u64_to_user_ptr(copy.syncs); 763 kperfmon_ids = u64_to_user_ptr(copy.kperfmon_ids); 764 765 for (i = 0; i < copy.count; i++) { 766 u32 sync; 767 u64 ids; 768 u32 __user *ids_pointer; 769 u32 id; 770 771 if (copy_from_user(&sync, syncs++, sizeof(sync))) { 772 err = -EFAULT; 773 goto error; 774 } 775 776 if (copy_from_user(&ids, kperfmon_ids++, sizeof(ids))) { 777 err = -EFAULT; 778 goto error; 779 } 780 781 ids_pointer = u64_to_user_ptr(ids); 782 783 for (j = 0; j < copy.nperfmons; j++) { 784 if (copy_from_user(&id, ids_pointer++, sizeof(id))) { 785 err = -EFAULT; 786 goto error; 787 } 788 789 job->performance_query.queries[i].kperfmon_ids[j] = id; 790 } 791 792 job->performance_query.queries[i].syncobj = drm_syncobj_find(file_priv, sync); 793 } 794 job->performance_query.count = copy.count; 795 job->performance_query.nperfmons = copy.nperfmons; 796 job->performance_query.ncounters = copy.ncounters; 797 798 job->copy.do_64bit = copy.do_64bit; 799 job->copy.do_partial = copy.do_partial; 800 job->copy.availability_bit = copy.availability_bit; 801 job->copy.offset = copy.offset; 802 job->copy.stride = copy.stride; 803 804 return 0; 805 806 error: 807 v3d_performance_query_info_free(&job->performance_query, i); 808 return err; 809 } 810 811 /* Whenever userspace sets ioctl extensions, v3d_get_extensions parses data 812 * according to the extension id (name). 813 */ 814 static int 815 v3d_get_extensions(struct drm_file *file_priv, 816 u64 ext_handles, 817 struct v3d_submit_ext *se, 818 struct v3d_cpu_job *job) 819 { 820 struct drm_v3d_extension __user *user_ext; 821 int ret; 822 823 user_ext = u64_to_user_ptr(ext_handles); 824 while (user_ext) { 825 struct drm_v3d_extension ext; 826 827 if (copy_from_user(&ext, user_ext, sizeof(ext))) { 828 DRM_DEBUG("Failed to copy submit extension\n"); 829 return -EFAULT; 830 } 831 832 switch (ext.id) { 833 case DRM_V3D_EXT_ID_MULTI_SYNC: 834 ret = v3d_get_multisync_submit_deps(file_priv, user_ext, se); 835 break; 836 case DRM_V3D_EXT_ID_CPU_INDIRECT_CSD: 837 ret = v3d_get_cpu_indirect_csd_params(file_priv, user_ext, job); 838 break; 839 case DRM_V3D_EXT_ID_CPU_TIMESTAMP_QUERY: 840 ret = v3d_get_cpu_timestamp_query_params(file_priv, user_ext, job); 841 break; 842 case DRM_V3D_EXT_ID_CPU_RESET_TIMESTAMP_QUERY: 843 ret = v3d_get_cpu_reset_timestamp_params(file_priv, user_ext, job); 844 break; 845 case DRM_V3D_EXT_ID_CPU_COPY_TIMESTAMP_QUERY: 846 ret = v3d_get_cpu_copy_query_results_params(file_priv, user_ext, job); 847 break; 848 case DRM_V3D_EXT_ID_CPU_RESET_PERFORMANCE_QUERY: 849 ret = v3d_get_cpu_reset_performance_params(file_priv, user_ext, job); 850 break; 851 case DRM_V3D_EXT_ID_CPU_COPY_PERFORMANCE_QUERY: 852 ret = v3d_get_cpu_copy_performance_query_params(file_priv, user_ext, job); 853 break; 854 default: 855 DRM_DEBUG_DRIVER("Unknown extension id: %d\n", ext.id); 856 return -EINVAL; 857 } 858 859 if (ret) 860 return ret; 861 862 user_ext = u64_to_user_ptr(ext.next); 863 } 864 865 return 0; 866 } 867 868 /** 869 * v3d_submit_cl_ioctl() - Submits a job (frame) to the V3D. 870 * @dev: DRM device 871 * @data: ioctl argument 872 * @file_priv: DRM file for this fd 873 * 874 * This is the main entrypoint for userspace to submit a 3D frame to 875 * the GPU. Userspace provides the binner command list (if 876 * applicable), and the kernel sets up the render command list to draw 877 * to the framebuffer described in the ioctl, using the command lists 878 * that the 3D engine's binner will produce. 879 */ 880 int 881 v3d_submit_cl_ioctl(struct drm_device *dev, void *data, 882 struct drm_file *file_priv) 883 { 884 struct v3d_dev *v3d = to_v3d_dev(dev); 885 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 886 struct drm_v3d_submit_cl *args = data; 887 struct v3d_submit_ext se = {0}; 888 struct v3d_bin_job *bin = NULL; 889 struct v3d_render_job *render = NULL; 890 struct v3d_job *clean_job = NULL; 891 struct v3d_job *last_job; 892 struct ww_acquire_ctx acquire_ctx; 893 int ret = 0; 894 895 trace_v3d_submit_cl_ioctl(&v3d->drm, args->rcl_start, args->rcl_end); 896 897 if (args->pad) 898 return -EINVAL; 899 900 if (args->flags && 901 args->flags & ~(DRM_V3D_SUBMIT_CL_FLUSH_CACHE | 902 DRM_V3D_SUBMIT_EXTENSION)) { 903 DRM_INFO("invalid flags: %d\n", args->flags); 904 return -EINVAL; 905 } 906 907 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) { 908 ret = v3d_get_extensions(file_priv, args->extensions, &se, NULL); 909 if (ret) { 910 DRM_DEBUG("Failed to get extensions.\n"); 911 return ret; 912 } 913 } 914 915 ret = v3d_job_allocate((void *)&render, sizeof(*render)); 916 if (ret) 917 return ret; 918 919 ret = v3d_job_init(v3d, file_priv, &render->base, 920 v3d_render_job_free, args->in_sync_rcl, &se, V3D_RENDER); 921 if (ret) { 922 v3d_job_deallocate((void *)&render); 923 goto fail; 924 } 925 926 render->start = args->rcl_start; 927 render->end = args->rcl_end; 928 INIT_LIST_HEAD(&render->unref_list); 929 930 if (args->bcl_start != args->bcl_end) { 931 ret = v3d_job_allocate((void *)&bin, sizeof(*bin)); 932 if (ret) 933 goto fail; 934 935 ret = v3d_job_init(v3d, file_priv, &bin->base, 936 v3d_job_free, args->in_sync_bcl, &se, V3D_BIN); 937 if (ret) { 938 v3d_job_deallocate((void *)&bin); 939 goto fail; 940 } 941 942 bin->start = args->bcl_start; 943 bin->end = args->bcl_end; 944 bin->qma = args->qma; 945 bin->qms = args->qms; 946 bin->qts = args->qts; 947 bin->render = render; 948 } 949 950 if (args->flags & DRM_V3D_SUBMIT_CL_FLUSH_CACHE) { 951 ret = v3d_job_allocate((void *)&clean_job, sizeof(*clean_job)); 952 if (ret) 953 goto fail; 954 955 ret = v3d_job_init(v3d, file_priv, clean_job, 956 v3d_job_free, 0, NULL, V3D_CACHE_CLEAN); 957 if (ret) { 958 v3d_job_deallocate((void *)&clean_job); 959 goto fail; 960 } 961 962 last_job = clean_job; 963 } else { 964 last_job = &render->base; 965 } 966 967 ret = v3d_lookup_bos(dev, file_priv, last_job, 968 args->bo_handles, args->bo_handle_count); 969 if (ret) 970 goto fail; 971 972 ret = v3d_lock_bo_reservations(last_job, &acquire_ctx); 973 if (ret) 974 goto fail; 975 976 if (args->perfmon_id) { 977 render->base.perfmon = v3d_perfmon_find(v3d_priv, 978 args->perfmon_id); 979 980 if (!render->base.perfmon) { 981 ret = -ENOENT; 982 goto fail_perfmon; 983 } 984 } 985 986 mutex_lock(&v3d->sched_lock); 987 if (bin) { 988 bin->base.perfmon = render->base.perfmon; 989 v3d_perfmon_get(bin->base.perfmon); 990 v3d_push_job(&bin->base); 991 992 ret = drm_sched_job_add_dependency(&render->base.base, 993 dma_fence_get(bin->base.done_fence)); 994 if (ret) 995 goto fail_unreserve; 996 } 997 998 v3d_push_job(&render->base); 999 1000 if (clean_job) { 1001 struct dma_fence *render_fence = 1002 dma_fence_get(render->base.done_fence); 1003 ret = drm_sched_job_add_dependency(&clean_job->base, 1004 render_fence); 1005 if (ret) 1006 goto fail_unreserve; 1007 clean_job->perfmon = render->base.perfmon; 1008 v3d_perfmon_get(clean_job->perfmon); 1009 v3d_push_job(clean_job); 1010 } 1011 1012 mutex_unlock(&v3d->sched_lock); 1013 1014 v3d_attach_fences_and_unlock_reservation(file_priv, 1015 last_job, 1016 &acquire_ctx, 1017 args->out_sync, 1018 &se, 1019 last_job->done_fence); 1020 1021 v3d_job_put(&bin->base); 1022 v3d_job_put(&render->base); 1023 v3d_job_put(clean_job); 1024 1025 return 0; 1026 1027 fail_unreserve: 1028 mutex_unlock(&v3d->sched_lock); 1029 fail_perfmon: 1030 drm_gem_unlock_reservations(last_job->bo, 1031 last_job->bo_count, &acquire_ctx); 1032 fail: 1033 v3d_job_cleanup((void *)bin); 1034 v3d_job_cleanup((void *)render); 1035 v3d_job_cleanup(clean_job); 1036 v3d_put_multisync_post_deps(&se); 1037 1038 return ret; 1039 } 1040 1041 /** 1042 * v3d_submit_tfu_ioctl() - Submits a TFU (texture formatting) job to the V3D. 1043 * @dev: DRM device 1044 * @data: ioctl argument 1045 * @file_priv: DRM file for this fd 1046 * 1047 * Userspace provides the register setup for the TFU, which we don't 1048 * need to validate since the TFU is behind the MMU. 1049 */ 1050 int 1051 v3d_submit_tfu_ioctl(struct drm_device *dev, void *data, 1052 struct drm_file *file_priv) 1053 { 1054 struct v3d_dev *v3d = to_v3d_dev(dev); 1055 struct drm_v3d_submit_tfu *args = data; 1056 struct v3d_submit_ext se = {0}; 1057 struct v3d_tfu_job *job = NULL; 1058 struct ww_acquire_ctx acquire_ctx; 1059 int ret = 0; 1060 1061 trace_v3d_submit_tfu_ioctl(&v3d->drm, args->iia); 1062 1063 if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) { 1064 DRM_DEBUG("invalid flags: %d\n", args->flags); 1065 return -EINVAL; 1066 } 1067 1068 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) { 1069 ret = v3d_get_extensions(file_priv, args->extensions, &se, NULL); 1070 if (ret) { 1071 DRM_DEBUG("Failed to get extensions.\n"); 1072 return ret; 1073 } 1074 } 1075 1076 ret = v3d_job_allocate((void *)&job, sizeof(*job)); 1077 if (ret) 1078 return ret; 1079 1080 ret = v3d_job_init(v3d, file_priv, &job->base, 1081 v3d_job_free, args->in_sync, &se, V3D_TFU); 1082 if (ret) { 1083 v3d_job_deallocate((void *)&job); 1084 goto fail; 1085 } 1086 1087 job->base.bo = kcalloc(ARRAY_SIZE(args->bo_handles), 1088 sizeof(*job->base.bo), GFP_KERNEL); 1089 if (!job->base.bo) { 1090 ret = -ENOMEM; 1091 goto fail; 1092 } 1093 1094 job->args = *args; 1095 1096 for (job->base.bo_count = 0; 1097 job->base.bo_count < ARRAY_SIZE(args->bo_handles); 1098 job->base.bo_count++) { 1099 struct drm_gem_object *bo; 1100 1101 if (!args->bo_handles[job->base.bo_count]) 1102 break; 1103 1104 bo = drm_gem_object_lookup(file_priv, args->bo_handles[job->base.bo_count]); 1105 if (!bo) { 1106 DRM_DEBUG("Failed to look up GEM BO %d: %d\n", 1107 job->base.bo_count, 1108 args->bo_handles[job->base.bo_count]); 1109 ret = -ENOENT; 1110 goto fail; 1111 } 1112 job->base.bo[job->base.bo_count] = bo; 1113 } 1114 1115 ret = v3d_lock_bo_reservations(&job->base, &acquire_ctx); 1116 if (ret) 1117 goto fail; 1118 1119 mutex_lock(&v3d->sched_lock); 1120 v3d_push_job(&job->base); 1121 mutex_unlock(&v3d->sched_lock); 1122 1123 v3d_attach_fences_and_unlock_reservation(file_priv, 1124 &job->base, &acquire_ctx, 1125 args->out_sync, 1126 &se, 1127 job->base.done_fence); 1128 1129 v3d_job_put(&job->base); 1130 1131 return 0; 1132 1133 fail: 1134 v3d_job_cleanup((void *)job); 1135 v3d_put_multisync_post_deps(&se); 1136 1137 return ret; 1138 } 1139 1140 /** 1141 * v3d_submit_csd_ioctl() - Submits a CSD (compute shader) job to the V3D. 1142 * @dev: DRM device 1143 * @data: ioctl argument 1144 * @file_priv: DRM file for this fd 1145 * 1146 * Userspace provides the register setup for the CSD, which we don't 1147 * need to validate since the CSD is behind the MMU. 1148 */ 1149 int 1150 v3d_submit_csd_ioctl(struct drm_device *dev, void *data, 1151 struct drm_file *file_priv) 1152 { 1153 struct v3d_dev *v3d = to_v3d_dev(dev); 1154 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 1155 struct drm_v3d_submit_csd *args = data; 1156 struct v3d_submit_ext se = {0}; 1157 struct v3d_csd_job *job = NULL; 1158 struct v3d_job *clean_job = NULL; 1159 struct ww_acquire_ctx acquire_ctx; 1160 int ret; 1161 1162 trace_v3d_submit_csd_ioctl(&v3d->drm, args->cfg[5], args->cfg[6]); 1163 1164 if (args->pad) 1165 return -EINVAL; 1166 1167 if (!v3d_has_csd(v3d)) { 1168 DRM_DEBUG("Attempting CSD submit on non-CSD hardware\n"); 1169 return -EINVAL; 1170 } 1171 1172 if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) { 1173 DRM_INFO("invalid flags: %d\n", args->flags); 1174 return -EINVAL; 1175 } 1176 1177 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) { 1178 ret = v3d_get_extensions(file_priv, args->extensions, &se, NULL); 1179 if (ret) { 1180 DRM_DEBUG("Failed to get extensions.\n"); 1181 return ret; 1182 } 1183 } 1184 1185 ret = v3d_setup_csd_jobs_and_bos(file_priv, v3d, args, 1186 &job, &clean_job, &se, 1187 &acquire_ctx); 1188 if (ret) 1189 goto fail; 1190 1191 if (args->perfmon_id) { 1192 job->base.perfmon = v3d_perfmon_find(v3d_priv, 1193 args->perfmon_id); 1194 if (!job->base.perfmon) { 1195 ret = -ENOENT; 1196 goto fail_perfmon; 1197 } 1198 } 1199 1200 mutex_lock(&v3d->sched_lock); 1201 v3d_push_job(&job->base); 1202 1203 ret = drm_sched_job_add_dependency(&clean_job->base, 1204 dma_fence_get(job->base.done_fence)); 1205 if (ret) 1206 goto fail_unreserve; 1207 1208 v3d_push_job(clean_job); 1209 mutex_unlock(&v3d->sched_lock); 1210 1211 v3d_attach_fences_and_unlock_reservation(file_priv, 1212 clean_job, 1213 &acquire_ctx, 1214 args->out_sync, 1215 &se, 1216 clean_job->done_fence); 1217 1218 v3d_job_put(&job->base); 1219 v3d_job_put(clean_job); 1220 1221 return 0; 1222 1223 fail_unreserve: 1224 mutex_unlock(&v3d->sched_lock); 1225 fail_perfmon: 1226 drm_gem_unlock_reservations(clean_job->bo, clean_job->bo_count, 1227 &acquire_ctx); 1228 fail: 1229 v3d_job_cleanup((void *)job); 1230 v3d_job_cleanup(clean_job); 1231 v3d_put_multisync_post_deps(&se); 1232 1233 return ret; 1234 } 1235 1236 static const unsigned int cpu_job_bo_handle_count[] = { 1237 [V3D_CPU_JOB_TYPE_INDIRECT_CSD] = 1, 1238 [V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY] = 1, 1239 [V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY] = 1, 1240 [V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY] = 2, 1241 [V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY] = 0, 1242 [V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY] = 1, 1243 }; 1244 1245 /** 1246 * v3d_submit_cpu_ioctl() - Submits a CPU job to the V3D. 1247 * @dev: DRM device 1248 * @data: ioctl argument 1249 * @file_priv: DRM file for this fd 1250 * 1251 * Userspace specifies the CPU job type and data required to perform its 1252 * operations through the drm_v3d_extension struct. 1253 */ 1254 int 1255 v3d_submit_cpu_ioctl(struct drm_device *dev, void *data, 1256 struct drm_file *file_priv) 1257 { 1258 struct v3d_dev *v3d = to_v3d_dev(dev); 1259 struct drm_v3d_submit_cpu *args = data; 1260 struct v3d_submit_ext se = {0}; 1261 struct v3d_submit_ext *out_se = NULL; 1262 struct v3d_cpu_job *cpu_job = NULL; 1263 struct v3d_csd_job *csd_job = NULL; 1264 struct v3d_job *clean_job = NULL; 1265 struct ww_acquire_ctx acquire_ctx; 1266 int ret; 1267 1268 if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) { 1269 DRM_INFO("Invalid flags: %d\n", args->flags); 1270 return -EINVAL; 1271 } 1272 1273 ret = v3d_job_allocate((void *)&cpu_job, sizeof(*cpu_job)); 1274 if (ret) 1275 return ret; 1276 1277 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) { 1278 ret = v3d_get_extensions(file_priv, args->extensions, &se, cpu_job); 1279 if (ret) { 1280 DRM_DEBUG("Failed to get extensions.\n"); 1281 goto fail; 1282 } 1283 } 1284 1285 /* Every CPU job must have a CPU job user extension */ 1286 if (!cpu_job->job_type) { 1287 DRM_DEBUG("CPU job must have a CPU job user extension.\n"); 1288 ret = -EINVAL; 1289 goto fail; 1290 } 1291 1292 if (args->bo_handle_count != cpu_job_bo_handle_count[cpu_job->job_type]) { 1293 DRM_DEBUG("This CPU job was not submitted with the proper number of BOs.\n"); 1294 ret = -EINVAL; 1295 goto fail; 1296 } 1297 1298 trace_v3d_submit_cpu_ioctl(&v3d->drm, cpu_job->job_type); 1299 1300 ret = v3d_job_init(v3d, file_priv, &cpu_job->base, 1301 v3d_job_free, 0, &se, V3D_CPU); 1302 if (ret) { 1303 v3d_job_deallocate((void *)&cpu_job); 1304 goto fail; 1305 } 1306 1307 clean_job = cpu_job->indirect_csd.clean_job; 1308 csd_job = cpu_job->indirect_csd.job; 1309 1310 if (args->bo_handle_count) { 1311 ret = v3d_lookup_bos(dev, file_priv, &cpu_job->base, 1312 args->bo_handles, args->bo_handle_count); 1313 if (ret) 1314 goto fail; 1315 1316 ret = v3d_lock_bo_reservations(&cpu_job->base, &acquire_ctx); 1317 if (ret) 1318 goto fail; 1319 } 1320 1321 mutex_lock(&v3d->sched_lock); 1322 v3d_push_job(&cpu_job->base); 1323 1324 switch (cpu_job->job_type) { 1325 case V3D_CPU_JOB_TYPE_INDIRECT_CSD: 1326 ret = drm_sched_job_add_dependency(&csd_job->base.base, 1327 dma_fence_get(cpu_job->base.done_fence)); 1328 if (ret) 1329 goto fail_unreserve; 1330 1331 v3d_push_job(&csd_job->base); 1332 1333 ret = drm_sched_job_add_dependency(&clean_job->base, 1334 dma_fence_get(csd_job->base.done_fence)); 1335 if (ret) 1336 goto fail_unreserve; 1337 1338 v3d_push_job(clean_job); 1339 1340 break; 1341 default: 1342 break; 1343 } 1344 mutex_unlock(&v3d->sched_lock); 1345 1346 out_se = (cpu_job->job_type == V3D_CPU_JOB_TYPE_INDIRECT_CSD) ? NULL : &se; 1347 1348 v3d_attach_fences_and_unlock_reservation(file_priv, 1349 &cpu_job->base, 1350 &acquire_ctx, 0, 1351 out_se, cpu_job->base.done_fence); 1352 1353 switch (cpu_job->job_type) { 1354 case V3D_CPU_JOB_TYPE_INDIRECT_CSD: 1355 v3d_attach_fences_and_unlock_reservation(file_priv, 1356 clean_job, 1357 &cpu_job->indirect_csd.acquire_ctx, 1358 0, &se, clean_job->done_fence); 1359 break; 1360 default: 1361 break; 1362 } 1363 1364 v3d_job_put(&cpu_job->base); 1365 v3d_job_put(&csd_job->base); 1366 v3d_job_put(clean_job); 1367 1368 return 0; 1369 1370 fail_unreserve: 1371 mutex_unlock(&v3d->sched_lock); 1372 1373 drm_gem_unlock_reservations(cpu_job->base.bo, cpu_job->base.bo_count, 1374 &acquire_ctx); 1375 1376 drm_gem_unlock_reservations(clean_job->bo, clean_job->bo_count, 1377 &cpu_job->indirect_csd.acquire_ctx); 1378 1379 fail: 1380 v3d_job_cleanup((void *)cpu_job); 1381 v3d_job_cleanup((void *)csd_job); 1382 v3d_job_cleanup(clean_job); 1383 v3d_put_multisync_post_deps(&se); 1384 kvfree(cpu_job->timestamp_query.queries); 1385 kvfree(cpu_job->performance_query.queries); 1386 1387 return ret; 1388 } 1389