1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Tegra host1x driver 4 * 5 * Copyright (c) 2010-2013, NVIDIA Corporation. 6 */ 7 8 #include <linux/clk.h> 9 #include <linux/delay.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/io.h> 12 #include <linux/list.h> 13 #include <linux/module.h> 14 #include <linux/of_device.h> 15 #include <linux/of.h> 16 #include <linux/pm_runtime.h> 17 #include <linux/slab.h> 18 19 #include <soc/tegra/common.h> 20 21 #define CREATE_TRACE_POINTS 22 #include <trace/events/host1x.h> 23 #undef CREATE_TRACE_POINTS 24 25 #if IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU) 26 #include <asm/dma-iommu.h> 27 #endif 28 29 #include "bus.h" 30 #include "channel.h" 31 #include "context.h" 32 #include "debug.h" 33 #include "dev.h" 34 #include "intr.h" 35 36 #include "hw/host1x01.h" 37 #include "hw/host1x02.h" 38 #include "hw/host1x04.h" 39 #include "hw/host1x05.h" 40 #include "hw/host1x06.h" 41 #include "hw/host1x07.h" 42 43 void host1x_hypervisor_writel(struct host1x *host1x, u32 v, u32 r) 44 { 45 writel(v, host1x->hv_regs + r); 46 } 47 48 u32 host1x_hypervisor_readl(struct host1x *host1x, u32 r) 49 { 50 return readl(host1x->hv_regs + r); 51 } 52 53 void host1x_sync_writel(struct host1x *host1x, u32 v, u32 r) 54 { 55 void __iomem *sync_regs = host1x->regs + host1x->info->sync_offset; 56 57 writel(v, sync_regs + r); 58 } 59 60 u32 host1x_sync_readl(struct host1x *host1x, u32 r) 61 { 62 void __iomem *sync_regs = host1x->regs + host1x->info->sync_offset; 63 64 return readl(sync_regs + r); 65 } 66 67 void host1x_ch_writel(struct host1x_channel *ch, u32 v, u32 r) 68 { 69 writel(v, ch->regs + r); 70 } 71 72 u32 host1x_ch_readl(struct host1x_channel *ch, u32 r) 73 { 74 return readl(ch->regs + r); 75 } 76 77 static const struct host1x_info host1x01_info = { 78 .nb_channels = 8, 79 .nb_pts = 32, 80 .nb_mlocks = 16, 81 .nb_bases = 8, 82 .init = host1x01_init, 83 .sync_offset = 0x3000, 84 .dma_mask = DMA_BIT_MASK(32), 85 .has_wide_gather = false, 86 .has_hypervisor = false, 87 .num_sid_entries = 0, 88 .sid_table = NULL, 89 .reserve_vblank_syncpts = true, 90 }; 91 92 static const struct host1x_info host1x02_info = { 93 .nb_channels = 9, 94 .nb_pts = 32, 95 .nb_mlocks = 16, 96 .nb_bases = 12, 97 .init = host1x02_init, 98 .sync_offset = 0x3000, 99 .dma_mask = DMA_BIT_MASK(32), 100 .has_wide_gather = false, 101 .has_hypervisor = false, 102 .num_sid_entries = 0, 103 .sid_table = NULL, 104 .reserve_vblank_syncpts = true, 105 }; 106 107 static const struct host1x_info host1x04_info = { 108 .nb_channels = 12, 109 .nb_pts = 192, 110 .nb_mlocks = 16, 111 .nb_bases = 64, 112 .init = host1x04_init, 113 .sync_offset = 0x2100, 114 .dma_mask = DMA_BIT_MASK(34), 115 .has_wide_gather = false, 116 .has_hypervisor = false, 117 .num_sid_entries = 0, 118 .sid_table = NULL, 119 .reserve_vblank_syncpts = false, 120 }; 121 122 static const struct host1x_info host1x05_info = { 123 .nb_channels = 14, 124 .nb_pts = 192, 125 .nb_mlocks = 16, 126 .nb_bases = 64, 127 .init = host1x05_init, 128 .sync_offset = 0x2100, 129 .dma_mask = DMA_BIT_MASK(34), 130 .has_wide_gather = false, 131 .has_hypervisor = false, 132 .num_sid_entries = 0, 133 .sid_table = NULL, 134 .reserve_vblank_syncpts = false, 135 }; 136 137 static const struct host1x_sid_entry tegra186_sid_table[] = { 138 { 139 /* VIC */ 140 .base = 0x1af0, 141 .offset = 0x30, 142 .limit = 0x34 143 }, 144 { 145 /* NVDEC */ 146 .base = 0x1b00, 147 .offset = 0x30, 148 .limit = 0x34 149 }, 150 }; 151 152 static const struct host1x_info host1x06_info = { 153 .nb_channels = 63, 154 .nb_pts = 576, 155 .nb_mlocks = 24, 156 .nb_bases = 16, 157 .init = host1x06_init, 158 .sync_offset = 0x0, 159 .dma_mask = DMA_BIT_MASK(40), 160 .has_wide_gather = true, 161 .has_hypervisor = true, 162 .num_sid_entries = ARRAY_SIZE(tegra186_sid_table), 163 .sid_table = tegra186_sid_table, 164 .reserve_vblank_syncpts = false, 165 }; 166 167 static const struct host1x_sid_entry tegra194_sid_table[] = { 168 { 169 /* VIC */ 170 .base = 0x1af0, 171 .offset = 0x30, 172 .limit = 0x34 173 }, 174 { 175 /* NVDEC */ 176 .base = 0x1b00, 177 .offset = 0x30, 178 .limit = 0x34 179 }, 180 { 181 /* NVDEC1 */ 182 .base = 0x1bc0, 183 .offset = 0x30, 184 .limit = 0x34 185 }, 186 }; 187 188 static const struct host1x_info host1x07_info = { 189 .nb_channels = 63, 190 .nb_pts = 704, 191 .nb_mlocks = 32, 192 .nb_bases = 0, 193 .init = host1x07_init, 194 .sync_offset = 0x0, 195 .dma_mask = DMA_BIT_MASK(40), 196 .has_wide_gather = true, 197 .has_hypervisor = true, 198 .num_sid_entries = ARRAY_SIZE(tegra194_sid_table), 199 .sid_table = tegra194_sid_table, 200 .reserve_vblank_syncpts = false, 201 }; 202 203 static const struct of_device_id host1x_of_match[] = { 204 { .compatible = "nvidia,tegra194-host1x", .data = &host1x07_info, }, 205 { .compatible = "nvidia,tegra186-host1x", .data = &host1x06_info, }, 206 { .compatible = "nvidia,tegra210-host1x", .data = &host1x05_info, }, 207 { .compatible = "nvidia,tegra124-host1x", .data = &host1x04_info, }, 208 { .compatible = "nvidia,tegra114-host1x", .data = &host1x02_info, }, 209 { .compatible = "nvidia,tegra30-host1x", .data = &host1x01_info, }, 210 { .compatible = "nvidia,tegra20-host1x", .data = &host1x01_info, }, 211 { }, 212 }; 213 MODULE_DEVICE_TABLE(of, host1x_of_match); 214 215 static void host1x_setup_sid_table(struct host1x *host) 216 { 217 const struct host1x_info *info = host->info; 218 unsigned int i; 219 220 if (!info->has_hypervisor) 221 return; 222 223 for (i = 0; i < info->num_sid_entries; i++) { 224 const struct host1x_sid_entry *entry = &info->sid_table[i]; 225 226 host1x_hypervisor_writel(host, entry->offset, entry->base); 227 host1x_hypervisor_writel(host, entry->limit, entry->base + 4); 228 } 229 } 230 231 static bool host1x_wants_iommu(struct host1x *host1x) 232 { 233 /* 234 * If we support addressing a maximum of 32 bits of physical memory 235 * and if the host1x firewall is enabled, there's no need to enable 236 * IOMMU support. This can happen for example on Tegra20, Tegra30 237 * and Tegra114. 238 * 239 * Tegra124 and later can address up to 34 bits of physical memory and 240 * many platforms come equipped with more than 2 GiB of system memory, 241 * which requires crossing the 4 GiB boundary. But there's a catch: on 242 * SoCs before Tegra186 (i.e. Tegra124 and Tegra210), the host1x can 243 * only address up to 32 bits of memory in GATHER opcodes, which means 244 * that command buffers need to either be in the first 2 GiB of system 245 * memory (which could quickly lead to memory exhaustion), or command 246 * buffers need to be treated differently from other buffers (which is 247 * not possible with the current ABI). 248 * 249 * A third option is to use the IOMMU in these cases to make sure all 250 * buffers will be mapped into a 32-bit IOVA space that host1x can 251 * address. This allows all of the system memory to be used and works 252 * within the limitations of the host1x on these SoCs. 253 * 254 * In summary, default to enable IOMMU on Tegra124 and later. For any 255 * of the earlier SoCs, only use the IOMMU for additional safety when 256 * the host1x firewall is disabled. 257 */ 258 if (host1x->info->dma_mask <= DMA_BIT_MASK(32)) { 259 if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL)) 260 return false; 261 } 262 263 return true; 264 } 265 266 static struct iommu_domain *host1x_iommu_attach(struct host1x *host) 267 { 268 struct iommu_domain *domain = iommu_get_domain_for_dev(host->dev); 269 int err; 270 271 #if IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU) 272 if (host->dev->archdata.mapping) { 273 struct dma_iommu_mapping *mapping = 274 to_dma_iommu_mapping(host->dev); 275 arm_iommu_detach_device(host->dev); 276 arm_iommu_release_mapping(mapping); 277 278 domain = iommu_get_domain_for_dev(host->dev); 279 } 280 #endif 281 282 /* 283 * We may not always want to enable IOMMU support (for example if the 284 * host1x firewall is already enabled and we don't support addressing 285 * more than 32 bits of physical memory), so check for that first. 286 * 287 * Similarly, if host1x is already attached to an IOMMU (via the DMA 288 * API), don't try to attach again. 289 */ 290 if (!host1x_wants_iommu(host) || domain) 291 return domain; 292 293 host->group = iommu_group_get(host->dev); 294 if (host->group) { 295 struct iommu_domain_geometry *geometry; 296 dma_addr_t start, end; 297 unsigned long order; 298 299 err = iova_cache_get(); 300 if (err < 0) 301 goto put_group; 302 303 host->domain = iommu_domain_alloc(&platform_bus_type); 304 if (!host->domain) { 305 err = -ENOMEM; 306 goto put_cache; 307 } 308 309 err = iommu_attach_group(host->domain, host->group); 310 if (err) { 311 if (err == -ENODEV) 312 err = 0; 313 314 goto free_domain; 315 } 316 317 geometry = &host->domain->geometry; 318 start = geometry->aperture_start & host->info->dma_mask; 319 end = geometry->aperture_end & host->info->dma_mask; 320 321 order = __ffs(host->domain->pgsize_bitmap); 322 init_iova_domain(&host->iova, 1UL << order, start >> order); 323 host->iova_end = end; 324 325 domain = host->domain; 326 } 327 328 return domain; 329 330 free_domain: 331 iommu_domain_free(host->domain); 332 host->domain = NULL; 333 put_cache: 334 iova_cache_put(); 335 put_group: 336 iommu_group_put(host->group); 337 host->group = NULL; 338 339 return ERR_PTR(err); 340 } 341 342 static int host1x_iommu_init(struct host1x *host) 343 { 344 u64 mask = host->info->dma_mask; 345 struct iommu_domain *domain; 346 int err; 347 348 domain = host1x_iommu_attach(host); 349 if (IS_ERR(domain)) { 350 err = PTR_ERR(domain); 351 dev_err(host->dev, "failed to attach to IOMMU: %d\n", err); 352 return err; 353 } 354 355 /* 356 * If we're not behind an IOMMU make sure we don't get push buffers 357 * that are allocated outside of the range addressable by the GATHER 358 * opcode. 359 * 360 * Newer generations of Tegra (Tegra186 and later) support a wide 361 * variant of the GATHER opcode that allows addressing more bits. 362 */ 363 if (!domain && !host->info->has_wide_gather) 364 mask = DMA_BIT_MASK(32); 365 366 err = dma_coerce_mask_and_coherent(host->dev, mask); 367 if (err < 0) { 368 dev_err(host->dev, "failed to set DMA mask: %d\n", err); 369 return err; 370 } 371 372 return 0; 373 } 374 375 static void host1x_iommu_exit(struct host1x *host) 376 { 377 if (host->domain) { 378 put_iova_domain(&host->iova); 379 iommu_detach_group(host->domain, host->group); 380 381 iommu_domain_free(host->domain); 382 host->domain = NULL; 383 384 iova_cache_put(); 385 386 iommu_group_put(host->group); 387 host->group = NULL; 388 } 389 } 390 391 static int host1x_get_resets(struct host1x *host) 392 { 393 int err; 394 395 host->resets[0].id = "mc"; 396 host->resets[1].id = "host1x"; 397 host->nresets = ARRAY_SIZE(host->resets); 398 399 err = devm_reset_control_bulk_get_optional_exclusive_released( 400 host->dev, host->nresets, host->resets); 401 if (err) { 402 dev_err(host->dev, "failed to get reset: %d\n", err); 403 return err; 404 } 405 406 if (WARN_ON(!host->resets[1].rstc)) 407 return -ENOENT; 408 409 return 0; 410 } 411 412 static int host1x_probe(struct platform_device *pdev) 413 { 414 struct host1x *host; 415 struct resource *regs, *hv_regs = NULL; 416 int syncpt_irq; 417 int err; 418 419 host = devm_kzalloc(&pdev->dev, sizeof(*host), GFP_KERNEL); 420 if (!host) 421 return -ENOMEM; 422 423 host->info = of_device_get_match_data(&pdev->dev); 424 425 if (host->info->has_hypervisor) { 426 regs = platform_get_resource_byname(pdev, IORESOURCE_MEM, "vm"); 427 if (!regs) { 428 dev_err(&pdev->dev, "failed to get vm registers\n"); 429 return -ENXIO; 430 } 431 432 hv_regs = platform_get_resource_byname(pdev, IORESOURCE_MEM, 433 "hypervisor"); 434 if (!hv_regs) { 435 dev_err(&pdev->dev, 436 "failed to get hypervisor registers\n"); 437 return -ENXIO; 438 } 439 } else { 440 regs = platform_get_resource(pdev, IORESOURCE_MEM, 0); 441 if (!regs) { 442 dev_err(&pdev->dev, "failed to get registers\n"); 443 return -ENXIO; 444 } 445 } 446 447 syncpt_irq = platform_get_irq(pdev, 0); 448 if (syncpt_irq < 0) 449 return syncpt_irq; 450 451 mutex_init(&host->devices_lock); 452 INIT_LIST_HEAD(&host->devices); 453 INIT_LIST_HEAD(&host->list); 454 host->dev = &pdev->dev; 455 456 /* set common host1x device data */ 457 platform_set_drvdata(pdev, host); 458 459 host->regs = devm_ioremap_resource(&pdev->dev, regs); 460 if (IS_ERR(host->regs)) 461 return PTR_ERR(host->regs); 462 463 if (host->info->has_hypervisor) { 464 host->hv_regs = devm_ioremap_resource(&pdev->dev, hv_regs); 465 if (IS_ERR(host->hv_regs)) 466 return PTR_ERR(host->hv_regs); 467 } 468 469 host->dev->dma_parms = &host->dma_parms; 470 dma_set_max_seg_size(host->dev, UINT_MAX); 471 472 if (host->info->init) { 473 err = host->info->init(host); 474 if (err) 475 return err; 476 } 477 478 host->clk = devm_clk_get(&pdev->dev, NULL); 479 if (IS_ERR(host->clk)) { 480 err = PTR_ERR(host->clk); 481 482 if (err != -EPROBE_DEFER) 483 dev_err(&pdev->dev, "failed to get clock: %d\n", err); 484 485 return err; 486 } 487 488 err = host1x_get_resets(host); 489 if (err) 490 return err; 491 492 host1x_bo_cache_init(&host->cache); 493 494 err = host1x_iommu_init(host); 495 if (err < 0) { 496 dev_err(&pdev->dev, "failed to setup IOMMU: %d\n", err); 497 goto destroy_cache; 498 } 499 500 err = host1x_channel_list_init(&host->channel_list, 501 host->info->nb_channels); 502 if (err) { 503 dev_err(&pdev->dev, "failed to initialize channel list\n"); 504 goto iommu_exit; 505 } 506 507 err = host1x_memory_context_list_init(host); 508 if (err) { 509 dev_err(&pdev->dev, "failed to initialize context list\n"); 510 goto free_channels; 511 } 512 513 err = host1x_syncpt_init(host); 514 if (err) { 515 dev_err(&pdev->dev, "failed to initialize syncpts\n"); 516 goto free_contexts; 517 } 518 519 err = host1x_intr_init(host, syncpt_irq); 520 if (err) { 521 dev_err(&pdev->dev, "failed to initialize interrupts\n"); 522 goto deinit_syncpt; 523 } 524 525 pm_runtime_enable(&pdev->dev); 526 527 err = devm_tegra_core_dev_init_opp_table_common(&pdev->dev); 528 if (err) 529 goto pm_disable; 530 531 /* the driver's code isn't ready yet for the dynamic RPM */ 532 err = pm_runtime_resume_and_get(&pdev->dev); 533 if (err) 534 goto pm_disable; 535 536 host1x_debug_init(host); 537 538 err = host1x_register(host); 539 if (err < 0) 540 goto deinit_debugfs; 541 542 err = devm_of_platform_populate(&pdev->dev); 543 if (err < 0) 544 goto unregister; 545 546 return 0; 547 548 unregister: 549 host1x_unregister(host); 550 deinit_debugfs: 551 host1x_debug_deinit(host); 552 553 pm_runtime_put_sync_suspend(&pdev->dev); 554 pm_disable: 555 pm_runtime_disable(&pdev->dev); 556 557 host1x_intr_deinit(host); 558 deinit_syncpt: 559 host1x_syncpt_deinit(host); 560 free_contexts: 561 host1x_memory_context_list_free(&host->context_list); 562 free_channels: 563 host1x_channel_list_free(&host->channel_list); 564 iommu_exit: 565 host1x_iommu_exit(host); 566 destroy_cache: 567 host1x_bo_cache_destroy(&host->cache); 568 569 return err; 570 } 571 572 static int host1x_remove(struct platform_device *pdev) 573 { 574 struct host1x *host = platform_get_drvdata(pdev); 575 576 host1x_unregister(host); 577 host1x_debug_deinit(host); 578 579 pm_runtime_force_suspend(&pdev->dev); 580 581 host1x_intr_deinit(host); 582 host1x_syncpt_deinit(host); 583 host1x_memory_context_list_free(&host->context_list); 584 host1x_channel_list_free(&host->channel_list); 585 host1x_iommu_exit(host); 586 host1x_bo_cache_destroy(&host->cache); 587 588 return 0; 589 } 590 591 static int __maybe_unused host1x_runtime_suspend(struct device *dev) 592 { 593 struct host1x *host = dev_get_drvdata(dev); 594 int err; 595 596 host1x_intr_stop(host); 597 host1x_syncpt_save(host); 598 599 err = reset_control_bulk_assert(host->nresets, host->resets); 600 if (err) { 601 dev_err(dev, "failed to assert reset: %d\n", err); 602 goto resume_host1x; 603 } 604 605 usleep_range(1000, 2000); 606 607 clk_disable_unprepare(host->clk); 608 reset_control_bulk_release(host->nresets, host->resets); 609 610 return 0; 611 612 resume_host1x: 613 host1x_setup_sid_table(host); 614 host1x_syncpt_restore(host); 615 host1x_intr_start(host); 616 617 return err; 618 } 619 620 static int __maybe_unused host1x_runtime_resume(struct device *dev) 621 { 622 struct host1x *host = dev_get_drvdata(dev); 623 int err; 624 625 err = reset_control_bulk_acquire(host->nresets, host->resets); 626 if (err) { 627 dev_err(dev, "failed to acquire reset: %d\n", err); 628 return err; 629 } 630 631 err = clk_prepare_enable(host->clk); 632 if (err) { 633 dev_err(dev, "failed to enable clock: %d\n", err); 634 goto release_reset; 635 } 636 637 err = reset_control_bulk_deassert(host->nresets, host->resets); 638 if (err < 0) { 639 dev_err(dev, "failed to deassert reset: %d\n", err); 640 goto disable_clk; 641 } 642 643 host1x_setup_sid_table(host); 644 host1x_syncpt_restore(host); 645 host1x_intr_start(host); 646 647 return 0; 648 649 disable_clk: 650 clk_disable_unprepare(host->clk); 651 release_reset: 652 reset_control_bulk_release(host->nresets, host->resets); 653 654 return err; 655 } 656 657 static const struct dev_pm_ops host1x_pm_ops = { 658 SET_RUNTIME_PM_OPS(host1x_runtime_suspend, host1x_runtime_resume, 659 NULL) 660 /* TODO: add system suspend-resume once driver will be ready for that */ 661 }; 662 663 static struct platform_driver tegra_host1x_driver = { 664 .driver = { 665 .name = "tegra-host1x", 666 .of_match_table = host1x_of_match, 667 .pm = &host1x_pm_ops, 668 }, 669 .probe = host1x_probe, 670 .remove = host1x_remove, 671 }; 672 673 static struct platform_driver * const drivers[] = { 674 &tegra_host1x_driver, 675 &tegra_mipi_driver, 676 }; 677 678 static int __init tegra_host1x_init(void) 679 { 680 int err; 681 682 err = bus_register(&host1x_bus_type); 683 if (err < 0) 684 return err; 685 686 err = platform_register_drivers(drivers, ARRAY_SIZE(drivers)); 687 if (err < 0) 688 bus_unregister(&host1x_bus_type); 689 690 return err; 691 } 692 module_init(tegra_host1x_init); 693 694 static void __exit tegra_host1x_exit(void) 695 { 696 platform_unregister_drivers(drivers, ARRAY_SIZE(drivers)); 697 bus_unregister(&host1x_bus_type); 698 } 699 module_exit(tegra_host1x_exit); 700 701 /** 702 * host1x_get_dma_mask() - query the supported DMA mask for host1x 703 * @host1x: host1x instance 704 * 705 * Note that this returns the supported DMA mask for host1x, which can be 706 * different from the applicable DMA mask under certain circumstances. 707 */ 708 u64 host1x_get_dma_mask(struct host1x *host1x) 709 { 710 return host1x->info->dma_mask; 711 } 712 EXPORT_SYMBOL(host1x_get_dma_mask); 713 714 MODULE_AUTHOR("Thierry Reding <[email protected]>"); 715 MODULE_AUTHOR("Terje Bergstrom <[email protected]>"); 716 MODULE_DESCRIPTION("Host1x driver for Tegra products"); 717 MODULE_LICENSE("GPL"); 718