1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Tegra host1x driver 4 * 5 * Copyright (c) 2010-2013, NVIDIA Corporation. 6 */ 7 8 #include <linux/clk.h> 9 #include <linux/delay.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/io.h> 12 #include <linux/list.h> 13 #include <linux/module.h> 14 #include <linux/of_device.h> 15 #include <linux/of.h> 16 #include <linux/pm_runtime.h> 17 #include <linux/slab.h> 18 19 #include <soc/tegra/common.h> 20 21 #define CREATE_TRACE_POINTS 22 #include <trace/events/host1x.h> 23 #undef CREATE_TRACE_POINTS 24 25 #if IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU) 26 #include <asm/dma-iommu.h> 27 #endif 28 29 #include "bus.h" 30 #include "channel.h" 31 #include "context.h" 32 #include "debug.h" 33 #include "dev.h" 34 #include "intr.h" 35 36 #include "hw/host1x01.h" 37 #include "hw/host1x02.h" 38 #include "hw/host1x04.h" 39 #include "hw/host1x05.h" 40 #include "hw/host1x06.h" 41 #include "hw/host1x07.h" 42 43 void host1x_common_writel(struct host1x *host1x, u32 v, u32 r) 44 { 45 writel(v, host1x->common_regs + r); 46 } 47 48 void host1x_hypervisor_writel(struct host1x *host1x, u32 v, u32 r) 49 { 50 writel(v, host1x->hv_regs + r); 51 } 52 53 u32 host1x_hypervisor_readl(struct host1x *host1x, u32 r) 54 { 55 return readl(host1x->hv_regs + r); 56 } 57 58 void host1x_sync_writel(struct host1x *host1x, u32 v, u32 r) 59 { 60 void __iomem *sync_regs = host1x->regs + host1x->info->sync_offset; 61 62 writel(v, sync_regs + r); 63 } 64 65 u32 host1x_sync_readl(struct host1x *host1x, u32 r) 66 { 67 void __iomem *sync_regs = host1x->regs + host1x->info->sync_offset; 68 69 return readl(sync_regs + r); 70 } 71 72 void host1x_ch_writel(struct host1x_channel *ch, u32 v, u32 r) 73 { 74 writel(v, ch->regs + r); 75 } 76 77 u32 host1x_ch_readl(struct host1x_channel *ch, u32 r) 78 { 79 return readl(ch->regs + r); 80 } 81 82 static const struct host1x_info host1x01_info = { 83 .nb_channels = 8, 84 .nb_pts = 32, 85 .nb_mlocks = 16, 86 .nb_bases = 8, 87 .init = host1x01_init, 88 .sync_offset = 0x3000, 89 .dma_mask = DMA_BIT_MASK(32), 90 .has_wide_gather = false, 91 .has_hypervisor = false, 92 .num_sid_entries = 0, 93 .sid_table = NULL, 94 .reserve_vblank_syncpts = true, 95 }; 96 97 static const struct host1x_info host1x02_info = { 98 .nb_channels = 9, 99 .nb_pts = 32, 100 .nb_mlocks = 16, 101 .nb_bases = 12, 102 .init = host1x02_init, 103 .sync_offset = 0x3000, 104 .dma_mask = DMA_BIT_MASK(32), 105 .has_wide_gather = false, 106 .has_hypervisor = false, 107 .num_sid_entries = 0, 108 .sid_table = NULL, 109 .reserve_vblank_syncpts = true, 110 }; 111 112 static const struct host1x_info host1x04_info = { 113 .nb_channels = 12, 114 .nb_pts = 192, 115 .nb_mlocks = 16, 116 .nb_bases = 64, 117 .init = host1x04_init, 118 .sync_offset = 0x2100, 119 .dma_mask = DMA_BIT_MASK(34), 120 .has_wide_gather = false, 121 .has_hypervisor = false, 122 .num_sid_entries = 0, 123 .sid_table = NULL, 124 .reserve_vblank_syncpts = false, 125 }; 126 127 static const struct host1x_info host1x05_info = { 128 .nb_channels = 14, 129 .nb_pts = 192, 130 .nb_mlocks = 16, 131 .nb_bases = 64, 132 .init = host1x05_init, 133 .sync_offset = 0x2100, 134 .dma_mask = DMA_BIT_MASK(34), 135 .has_wide_gather = false, 136 .has_hypervisor = false, 137 .num_sid_entries = 0, 138 .sid_table = NULL, 139 .reserve_vblank_syncpts = false, 140 }; 141 142 static const struct host1x_sid_entry tegra186_sid_table[] = { 143 { 144 /* VIC */ 145 .base = 0x1af0, 146 .offset = 0x30, 147 .limit = 0x34 148 }, 149 { 150 /* NVDEC */ 151 .base = 0x1b00, 152 .offset = 0x30, 153 .limit = 0x34 154 }, 155 }; 156 157 static const struct host1x_info host1x06_info = { 158 .nb_channels = 63, 159 .nb_pts = 576, 160 .nb_mlocks = 24, 161 .nb_bases = 16, 162 .init = host1x06_init, 163 .sync_offset = 0x0, 164 .dma_mask = DMA_BIT_MASK(40), 165 .has_wide_gather = true, 166 .has_hypervisor = true, 167 .num_sid_entries = ARRAY_SIZE(tegra186_sid_table), 168 .sid_table = tegra186_sid_table, 169 .reserve_vblank_syncpts = false, 170 }; 171 172 static const struct host1x_sid_entry tegra194_sid_table[] = { 173 { 174 /* VIC */ 175 .base = 0x1af0, 176 .offset = 0x30, 177 .limit = 0x34 178 }, 179 { 180 /* NVDEC */ 181 .base = 0x1b00, 182 .offset = 0x30, 183 .limit = 0x34 184 }, 185 { 186 /* NVDEC1 */ 187 .base = 0x1bc0, 188 .offset = 0x30, 189 .limit = 0x34 190 }, 191 }; 192 193 static const struct host1x_info host1x07_info = { 194 .nb_channels = 63, 195 .nb_pts = 704, 196 .nb_mlocks = 32, 197 .nb_bases = 0, 198 .init = host1x07_init, 199 .sync_offset = 0x0, 200 .dma_mask = DMA_BIT_MASK(40), 201 .has_wide_gather = true, 202 .has_hypervisor = true, 203 .num_sid_entries = ARRAY_SIZE(tegra194_sid_table), 204 .sid_table = tegra194_sid_table, 205 .reserve_vblank_syncpts = false, 206 }; 207 208 static const struct of_device_id host1x_of_match[] = { 209 { .compatible = "nvidia,tegra194-host1x", .data = &host1x07_info, }, 210 { .compatible = "nvidia,tegra186-host1x", .data = &host1x06_info, }, 211 { .compatible = "nvidia,tegra210-host1x", .data = &host1x05_info, }, 212 { .compatible = "nvidia,tegra124-host1x", .data = &host1x04_info, }, 213 { .compatible = "nvidia,tegra114-host1x", .data = &host1x02_info, }, 214 { .compatible = "nvidia,tegra30-host1x", .data = &host1x01_info, }, 215 { .compatible = "nvidia,tegra20-host1x", .data = &host1x01_info, }, 216 { }, 217 }; 218 MODULE_DEVICE_TABLE(of, host1x_of_match); 219 220 static void host1x_setup_sid_table(struct host1x *host) 221 { 222 const struct host1x_info *info = host->info; 223 unsigned int i; 224 225 if (!info->has_hypervisor) 226 return; 227 228 for (i = 0; i < info->num_sid_entries; i++) { 229 const struct host1x_sid_entry *entry = &info->sid_table[i]; 230 231 host1x_hypervisor_writel(host, entry->offset, entry->base); 232 host1x_hypervisor_writel(host, entry->limit, entry->base + 4); 233 } 234 } 235 236 static bool host1x_wants_iommu(struct host1x *host1x) 237 { 238 /* 239 * If we support addressing a maximum of 32 bits of physical memory 240 * and if the host1x firewall is enabled, there's no need to enable 241 * IOMMU support. This can happen for example on Tegra20, Tegra30 242 * and Tegra114. 243 * 244 * Tegra124 and later can address up to 34 bits of physical memory and 245 * many platforms come equipped with more than 2 GiB of system memory, 246 * which requires crossing the 4 GiB boundary. But there's a catch: on 247 * SoCs before Tegra186 (i.e. Tegra124 and Tegra210), the host1x can 248 * only address up to 32 bits of memory in GATHER opcodes, which means 249 * that command buffers need to either be in the first 2 GiB of system 250 * memory (which could quickly lead to memory exhaustion), or command 251 * buffers need to be treated differently from other buffers (which is 252 * not possible with the current ABI). 253 * 254 * A third option is to use the IOMMU in these cases to make sure all 255 * buffers will be mapped into a 32-bit IOVA space that host1x can 256 * address. This allows all of the system memory to be used and works 257 * within the limitations of the host1x on these SoCs. 258 * 259 * In summary, default to enable IOMMU on Tegra124 and later. For any 260 * of the earlier SoCs, only use the IOMMU for additional safety when 261 * the host1x firewall is disabled. 262 */ 263 if (host1x->info->dma_mask <= DMA_BIT_MASK(32)) { 264 if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL)) 265 return false; 266 } 267 268 return true; 269 } 270 271 static struct iommu_domain *host1x_iommu_attach(struct host1x *host) 272 { 273 struct iommu_domain *domain = iommu_get_domain_for_dev(host->dev); 274 int err; 275 276 #if IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU) 277 if (host->dev->archdata.mapping) { 278 struct dma_iommu_mapping *mapping = 279 to_dma_iommu_mapping(host->dev); 280 arm_iommu_detach_device(host->dev); 281 arm_iommu_release_mapping(mapping); 282 283 domain = iommu_get_domain_for_dev(host->dev); 284 } 285 #endif 286 287 /* 288 * We may not always want to enable IOMMU support (for example if the 289 * host1x firewall is already enabled and we don't support addressing 290 * more than 32 bits of physical memory), so check for that first. 291 * 292 * Similarly, if host1x is already attached to an IOMMU (via the DMA 293 * API), don't try to attach again. 294 */ 295 if (!host1x_wants_iommu(host) || domain) 296 return domain; 297 298 host->group = iommu_group_get(host->dev); 299 if (host->group) { 300 struct iommu_domain_geometry *geometry; 301 dma_addr_t start, end; 302 unsigned long order; 303 304 err = iova_cache_get(); 305 if (err < 0) 306 goto put_group; 307 308 host->domain = iommu_domain_alloc(&platform_bus_type); 309 if (!host->domain) { 310 err = -ENOMEM; 311 goto put_cache; 312 } 313 314 err = iommu_attach_group(host->domain, host->group); 315 if (err) { 316 if (err == -ENODEV) 317 err = 0; 318 319 goto free_domain; 320 } 321 322 geometry = &host->domain->geometry; 323 start = geometry->aperture_start & host->info->dma_mask; 324 end = geometry->aperture_end & host->info->dma_mask; 325 326 order = __ffs(host->domain->pgsize_bitmap); 327 init_iova_domain(&host->iova, 1UL << order, start >> order); 328 host->iova_end = end; 329 330 domain = host->domain; 331 } 332 333 return domain; 334 335 free_domain: 336 iommu_domain_free(host->domain); 337 host->domain = NULL; 338 put_cache: 339 iova_cache_put(); 340 put_group: 341 iommu_group_put(host->group); 342 host->group = NULL; 343 344 return ERR_PTR(err); 345 } 346 347 static int host1x_iommu_init(struct host1x *host) 348 { 349 u64 mask = host->info->dma_mask; 350 struct iommu_domain *domain; 351 int err; 352 353 domain = host1x_iommu_attach(host); 354 if (IS_ERR(domain)) { 355 err = PTR_ERR(domain); 356 dev_err(host->dev, "failed to attach to IOMMU: %d\n", err); 357 return err; 358 } 359 360 /* 361 * If we're not behind an IOMMU make sure we don't get push buffers 362 * that are allocated outside of the range addressable by the GATHER 363 * opcode. 364 * 365 * Newer generations of Tegra (Tegra186 and later) support a wide 366 * variant of the GATHER opcode that allows addressing more bits. 367 */ 368 if (!domain && !host->info->has_wide_gather) 369 mask = DMA_BIT_MASK(32); 370 371 err = dma_coerce_mask_and_coherent(host->dev, mask); 372 if (err < 0) { 373 dev_err(host->dev, "failed to set DMA mask: %d\n", err); 374 return err; 375 } 376 377 return 0; 378 } 379 380 static void host1x_iommu_exit(struct host1x *host) 381 { 382 if (host->domain) { 383 put_iova_domain(&host->iova); 384 iommu_detach_group(host->domain, host->group); 385 386 iommu_domain_free(host->domain); 387 host->domain = NULL; 388 389 iova_cache_put(); 390 391 iommu_group_put(host->group); 392 host->group = NULL; 393 } 394 } 395 396 static int host1x_get_resets(struct host1x *host) 397 { 398 int err; 399 400 host->resets[0].id = "mc"; 401 host->resets[1].id = "host1x"; 402 host->nresets = ARRAY_SIZE(host->resets); 403 404 err = devm_reset_control_bulk_get_optional_exclusive_released( 405 host->dev, host->nresets, host->resets); 406 if (err) { 407 dev_err(host->dev, "failed to get reset: %d\n", err); 408 return err; 409 } 410 411 if (WARN_ON(!host->resets[1].rstc)) 412 return -ENOENT; 413 414 return 0; 415 } 416 417 static int host1x_probe(struct platform_device *pdev) 418 { 419 struct host1x *host; 420 int syncpt_irq; 421 int err; 422 423 host = devm_kzalloc(&pdev->dev, sizeof(*host), GFP_KERNEL); 424 if (!host) 425 return -ENOMEM; 426 427 host->info = of_device_get_match_data(&pdev->dev); 428 429 if (host->info->has_hypervisor) { 430 host->regs = devm_platform_ioremap_resource_byname(pdev, "vm"); 431 if (IS_ERR(host->regs)) 432 return PTR_ERR(host->regs); 433 434 host->hv_regs = devm_platform_ioremap_resource_byname(pdev, "hypervisor"); 435 if (IS_ERR(host->hv_regs)) 436 return PTR_ERR(host->hv_regs); 437 438 if (host->info->has_common) { 439 host->common_regs = devm_platform_ioremap_resource_byname(pdev, "common"); 440 if (IS_ERR(host->common_regs)) 441 return PTR_ERR(host->common_regs); 442 } 443 } else { 444 host->regs = devm_platform_ioremap_resource(pdev, 0); 445 if (IS_ERR(host->regs)) 446 return PTR_ERR(host->regs); 447 } 448 449 syncpt_irq = platform_get_irq(pdev, 0); 450 if (syncpt_irq < 0) 451 return syncpt_irq; 452 453 mutex_init(&host->devices_lock); 454 INIT_LIST_HEAD(&host->devices); 455 INIT_LIST_HEAD(&host->list); 456 host->dev = &pdev->dev; 457 458 /* set common host1x device data */ 459 platform_set_drvdata(pdev, host); 460 461 host->dev->dma_parms = &host->dma_parms; 462 dma_set_max_seg_size(host->dev, UINT_MAX); 463 464 if (host->info->init) { 465 err = host->info->init(host); 466 if (err) 467 return err; 468 } 469 470 host->clk = devm_clk_get(&pdev->dev, NULL); 471 if (IS_ERR(host->clk)) { 472 err = PTR_ERR(host->clk); 473 474 if (err != -EPROBE_DEFER) 475 dev_err(&pdev->dev, "failed to get clock: %d\n", err); 476 477 return err; 478 } 479 480 err = host1x_get_resets(host); 481 if (err) 482 return err; 483 484 host1x_bo_cache_init(&host->cache); 485 486 err = host1x_iommu_init(host); 487 if (err < 0) { 488 dev_err(&pdev->dev, "failed to setup IOMMU: %d\n", err); 489 goto destroy_cache; 490 } 491 492 err = host1x_channel_list_init(&host->channel_list, 493 host->info->nb_channels); 494 if (err) { 495 dev_err(&pdev->dev, "failed to initialize channel list\n"); 496 goto iommu_exit; 497 } 498 499 err = host1x_memory_context_list_init(host); 500 if (err) { 501 dev_err(&pdev->dev, "failed to initialize context list\n"); 502 goto free_channels; 503 } 504 505 err = host1x_syncpt_init(host); 506 if (err) { 507 dev_err(&pdev->dev, "failed to initialize syncpts\n"); 508 goto free_contexts; 509 } 510 511 err = host1x_intr_init(host, syncpt_irq); 512 if (err) { 513 dev_err(&pdev->dev, "failed to initialize interrupts\n"); 514 goto deinit_syncpt; 515 } 516 517 pm_runtime_enable(&pdev->dev); 518 519 err = devm_tegra_core_dev_init_opp_table_common(&pdev->dev); 520 if (err) 521 goto pm_disable; 522 523 /* the driver's code isn't ready yet for the dynamic RPM */ 524 err = pm_runtime_resume_and_get(&pdev->dev); 525 if (err) 526 goto pm_disable; 527 528 host1x_debug_init(host); 529 530 err = host1x_register(host); 531 if (err < 0) 532 goto deinit_debugfs; 533 534 err = devm_of_platform_populate(&pdev->dev); 535 if (err < 0) 536 goto unregister; 537 538 return 0; 539 540 unregister: 541 host1x_unregister(host); 542 deinit_debugfs: 543 host1x_debug_deinit(host); 544 545 pm_runtime_put_sync_suspend(&pdev->dev); 546 pm_disable: 547 pm_runtime_disable(&pdev->dev); 548 549 host1x_intr_deinit(host); 550 deinit_syncpt: 551 host1x_syncpt_deinit(host); 552 free_contexts: 553 host1x_memory_context_list_free(&host->context_list); 554 free_channels: 555 host1x_channel_list_free(&host->channel_list); 556 iommu_exit: 557 host1x_iommu_exit(host); 558 destroy_cache: 559 host1x_bo_cache_destroy(&host->cache); 560 561 return err; 562 } 563 564 static int host1x_remove(struct platform_device *pdev) 565 { 566 struct host1x *host = platform_get_drvdata(pdev); 567 568 host1x_unregister(host); 569 host1x_debug_deinit(host); 570 571 pm_runtime_force_suspend(&pdev->dev); 572 573 host1x_intr_deinit(host); 574 host1x_syncpt_deinit(host); 575 host1x_memory_context_list_free(&host->context_list); 576 host1x_channel_list_free(&host->channel_list); 577 host1x_iommu_exit(host); 578 host1x_bo_cache_destroy(&host->cache); 579 580 return 0; 581 } 582 583 static int __maybe_unused host1x_runtime_suspend(struct device *dev) 584 { 585 struct host1x *host = dev_get_drvdata(dev); 586 int err; 587 588 host1x_intr_stop(host); 589 host1x_syncpt_save(host); 590 591 err = reset_control_bulk_assert(host->nresets, host->resets); 592 if (err) { 593 dev_err(dev, "failed to assert reset: %d\n", err); 594 goto resume_host1x; 595 } 596 597 usleep_range(1000, 2000); 598 599 clk_disable_unprepare(host->clk); 600 reset_control_bulk_release(host->nresets, host->resets); 601 602 return 0; 603 604 resume_host1x: 605 host1x_setup_sid_table(host); 606 host1x_syncpt_restore(host); 607 host1x_intr_start(host); 608 609 return err; 610 } 611 612 static int __maybe_unused host1x_runtime_resume(struct device *dev) 613 { 614 struct host1x *host = dev_get_drvdata(dev); 615 int err; 616 617 err = reset_control_bulk_acquire(host->nresets, host->resets); 618 if (err) { 619 dev_err(dev, "failed to acquire reset: %d\n", err); 620 return err; 621 } 622 623 err = clk_prepare_enable(host->clk); 624 if (err) { 625 dev_err(dev, "failed to enable clock: %d\n", err); 626 goto release_reset; 627 } 628 629 err = reset_control_bulk_deassert(host->nresets, host->resets); 630 if (err < 0) { 631 dev_err(dev, "failed to deassert reset: %d\n", err); 632 goto disable_clk; 633 } 634 635 host1x_setup_sid_table(host); 636 host1x_syncpt_restore(host); 637 host1x_intr_start(host); 638 639 return 0; 640 641 disable_clk: 642 clk_disable_unprepare(host->clk); 643 release_reset: 644 reset_control_bulk_release(host->nresets, host->resets); 645 646 return err; 647 } 648 649 static const struct dev_pm_ops host1x_pm_ops = { 650 SET_RUNTIME_PM_OPS(host1x_runtime_suspend, host1x_runtime_resume, 651 NULL) 652 /* TODO: add system suspend-resume once driver will be ready for that */ 653 }; 654 655 static struct platform_driver tegra_host1x_driver = { 656 .driver = { 657 .name = "tegra-host1x", 658 .of_match_table = host1x_of_match, 659 .pm = &host1x_pm_ops, 660 }, 661 .probe = host1x_probe, 662 .remove = host1x_remove, 663 }; 664 665 static struct platform_driver * const drivers[] = { 666 &tegra_host1x_driver, 667 &tegra_mipi_driver, 668 }; 669 670 static int __init tegra_host1x_init(void) 671 { 672 int err; 673 674 err = bus_register(&host1x_bus_type); 675 if (err < 0) 676 return err; 677 678 err = platform_register_drivers(drivers, ARRAY_SIZE(drivers)); 679 if (err < 0) 680 bus_unregister(&host1x_bus_type); 681 682 return err; 683 } 684 module_init(tegra_host1x_init); 685 686 static void __exit tegra_host1x_exit(void) 687 { 688 platform_unregister_drivers(drivers, ARRAY_SIZE(drivers)); 689 bus_unregister(&host1x_bus_type); 690 } 691 module_exit(tegra_host1x_exit); 692 693 /** 694 * host1x_get_dma_mask() - query the supported DMA mask for host1x 695 * @host1x: host1x instance 696 * 697 * Note that this returns the supported DMA mask for host1x, which can be 698 * different from the applicable DMA mask under certain circumstances. 699 */ 700 u64 host1x_get_dma_mask(struct host1x *host1x) 701 { 702 return host1x->info->dma_mask; 703 } 704 EXPORT_SYMBOL(host1x_get_dma_mask); 705 706 MODULE_AUTHOR("Thierry Reding <[email protected]>"); 707 MODULE_AUTHOR("Terje Bergstrom <[email protected]>"); 708 MODULE_DESCRIPTION("Host1x driver for Tegra products"); 709 MODULE_LICENSE("GPL"); 710