1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright © 2015 Intel Corporation. 4 * 5 * Authors: David Woodhouse <[email protected]> 6 */ 7 8 #include <linux/mmu_notifier.h> 9 #include <linux/sched.h> 10 #include <linux/sched/mm.h> 11 #include <linux/slab.h> 12 #include <linux/rculist.h> 13 #include <linux/pci.h> 14 #include <linux/pci-ats.h> 15 #include <linux/dmar.h> 16 #include <linux/interrupt.h> 17 #include <linux/mm_types.h> 18 #include <linux/xarray.h> 19 #include <asm/page.h> 20 #include <asm/fpu/api.h> 21 22 #include "iommu.h" 23 #include "pasid.h" 24 #include "perf.h" 25 #include "../iommu-sva.h" 26 #include "trace.h" 27 28 static irqreturn_t prq_event_thread(int irq, void *d); 29 30 static DEFINE_XARRAY_ALLOC(pasid_private_array); 31 static int pasid_private_add(ioasid_t pasid, void *priv) 32 { 33 return xa_alloc(&pasid_private_array, &pasid, priv, 34 XA_LIMIT(pasid, pasid), GFP_ATOMIC); 35 } 36 37 static void pasid_private_remove(ioasid_t pasid) 38 { 39 xa_erase(&pasid_private_array, pasid); 40 } 41 42 static void *pasid_private_find(ioasid_t pasid) 43 { 44 return xa_load(&pasid_private_array, pasid); 45 } 46 47 static struct intel_svm_dev * 48 svm_lookup_device_by_dev(struct intel_svm *svm, struct device *dev) 49 { 50 struct intel_svm_dev *sdev = NULL, *t; 51 52 rcu_read_lock(); 53 list_for_each_entry_rcu(t, &svm->devs, list) { 54 if (t->dev == dev) { 55 sdev = t; 56 break; 57 } 58 } 59 rcu_read_unlock(); 60 61 return sdev; 62 } 63 64 int intel_svm_enable_prq(struct intel_iommu *iommu) 65 { 66 struct iopf_queue *iopfq; 67 struct page *pages; 68 int irq, ret; 69 70 pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, PRQ_ORDER); 71 if (!pages) { 72 pr_warn("IOMMU: %s: Failed to allocate page request queue\n", 73 iommu->name); 74 return -ENOMEM; 75 } 76 iommu->prq = page_address(pages); 77 78 irq = dmar_alloc_hwirq(IOMMU_IRQ_ID_OFFSET_PRQ + iommu->seq_id, iommu->node, iommu); 79 if (irq <= 0) { 80 pr_err("IOMMU: %s: Failed to create IRQ vector for page request queue\n", 81 iommu->name); 82 ret = -EINVAL; 83 goto free_prq; 84 } 85 iommu->pr_irq = irq; 86 87 snprintf(iommu->iopfq_name, sizeof(iommu->iopfq_name), 88 "dmar%d-iopfq", iommu->seq_id); 89 iopfq = iopf_queue_alloc(iommu->iopfq_name); 90 if (!iopfq) { 91 pr_err("IOMMU: %s: Failed to allocate iopf queue\n", iommu->name); 92 ret = -ENOMEM; 93 goto free_hwirq; 94 } 95 iommu->iopf_queue = iopfq; 96 97 snprintf(iommu->prq_name, sizeof(iommu->prq_name), "dmar%d-prq", iommu->seq_id); 98 99 ret = request_threaded_irq(irq, NULL, prq_event_thread, IRQF_ONESHOT, 100 iommu->prq_name, iommu); 101 if (ret) { 102 pr_err("IOMMU: %s: Failed to request IRQ for page request queue\n", 103 iommu->name); 104 goto free_iopfq; 105 } 106 dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL); 107 dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL); 108 dmar_writeq(iommu->reg + DMAR_PQA_REG, virt_to_phys(iommu->prq) | PRQ_ORDER); 109 110 init_completion(&iommu->prq_complete); 111 112 return 0; 113 114 free_iopfq: 115 iopf_queue_free(iommu->iopf_queue); 116 iommu->iopf_queue = NULL; 117 free_hwirq: 118 dmar_free_hwirq(irq); 119 iommu->pr_irq = 0; 120 free_prq: 121 free_pages((unsigned long)iommu->prq, PRQ_ORDER); 122 iommu->prq = NULL; 123 124 return ret; 125 } 126 127 int intel_svm_finish_prq(struct intel_iommu *iommu) 128 { 129 dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL); 130 dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL); 131 dmar_writeq(iommu->reg + DMAR_PQA_REG, 0ULL); 132 133 if (iommu->pr_irq) { 134 free_irq(iommu->pr_irq, iommu); 135 dmar_free_hwirq(iommu->pr_irq); 136 iommu->pr_irq = 0; 137 } 138 139 if (iommu->iopf_queue) { 140 iopf_queue_free(iommu->iopf_queue); 141 iommu->iopf_queue = NULL; 142 } 143 144 free_pages((unsigned long)iommu->prq, PRQ_ORDER); 145 iommu->prq = NULL; 146 147 return 0; 148 } 149 150 void intel_svm_check(struct intel_iommu *iommu) 151 { 152 if (!pasid_supported(iommu)) 153 return; 154 155 if (cpu_feature_enabled(X86_FEATURE_GBPAGES) && 156 !cap_fl1gp_support(iommu->cap)) { 157 pr_err("%s SVM disabled, incompatible 1GB page capability\n", 158 iommu->name); 159 return; 160 } 161 162 if (cpu_feature_enabled(X86_FEATURE_LA57) && 163 !cap_fl5lp_support(iommu->cap)) { 164 pr_err("%s SVM disabled, incompatible paging mode\n", 165 iommu->name); 166 return; 167 } 168 169 iommu->flags |= VTD_FLAG_SVM_CAPABLE; 170 } 171 172 static void __flush_svm_range_dev(struct intel_svm *svm, 173 struct intel_svm_dev *sdev, 174 unsigned long address, 175 unsigned long pages, int ih) 176 { 177 struct device_domain_info *info = dev_iommu_priv_get(sdev->dev); 178 179 if (WARN_ON(!pages)) 180 return; 181 182 qi_flush_piotlb(sdev->iommu, sdev->did, svm->pasid, address, pages, ih); 183 if (info->ats_enabled) { 184 qi_flush_dev_iotlb_pasid(sdev->iommu, sdev->sid, info->pfsid, 185 svm->pasid, sdev->qdep, address, 186 order_base_2(pages)); 187 quirk_extra_dev_tlb_flush(info, address, order_base_2(pages), 188 svm->pasid, sdev->qdep); 189 } 190 } 191 192 static void intel_flush_svm_range_dev(struct intel_svm *svm, 193 struct intel_svm_dev *sdev, 194 unsigned long address, 195 unsigned long pages, int ih) 196 { 197 unsigned long shift = ilog2(__roundup_pow_of_two(pages)); 198 unsigned long align = (1ULL << (VTD_PAGE_SHIFT + shift)); 199 unsigned long start = ALIGN_DOWN(address, align); 200 unsigned long end = ALIGN(address + (pages << VTD_PAGE_SHIFT), align); 201 202 while (start < end) { 203 __flush_svm_range_dev(svm, sdev, start, align >> VTD_PAGE_SHIFT, ih); 204 start += align; 205 } 206 } 207 208 static void intel_flush_svm_range(struct intel_svm *svm, unsigned long address, 209 unsigned long pages, int ih) 210 { 211 struct intel_svm_dev *sdev; 212 213 rcu_read_lock(); 214 list_for_each_entry_rcu(sdev, &svm->devs, list) 215 intel_flush_svm_range_dev(svm, sdev, address, pages, ih); 216 rcu_read_unlock(); 217 } 218 219 static void intel_flush_svm_all(struct intel_svm *svm) 220 { 221 struct device_domain_info *info; 222 struct intel_svm_dev *sdev; 223 224 rcu_read_lock(); 225 list_for_each_entry_rcu(sdev, &svm->devs, list) { 226 info = dev_iommu_priv_get(sdev->dev); 227 228 qi_flush_piotlb(sdev->iommu, sdev->did, svm->pasid, 0, -1UL, 0); 229 if (info->ats_enabled) { 230 qi_flush_dev_iotlb_pasid(sdev->iommu, sdev->sid, info->pfsid, 231 svm->pasid, sdev->qdep, 232 0, 64 - VTD_PAGE_SHIFT); 233 quirk_extra_dev_tlb_flush(info, 0, 64 - VTD_PAGE_SHIFT, 234 svm->pasid, sdev->qdep); 235 } 236 } 237 rcu_read_unlock(); 238 } 239 240 /* Pages have been freed at this point */ 241 static void intel_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, 242 struct mm_struct *mm, 243 unsigned long start, unsigned long end) 244 { 245 struct intel_svm *svm = container_of(mn, struct intel_svm, notifier); 246 247 if (start == 0 && end == -1UL) { 248 intel_flush_svm_all(svm); 249 return; 250 } 251 252 intel_flush_svm_range(svm, start, 253 (end - start + PAGE_SIZE - 1) >> VTD_PAGE_SHIFT, 0); 254 } 255 256 static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) 257 { 258 struct intel_svm *svm = container_of(mn, struct intel_svm, notifier); 259 struct intel_svm_dev *sdev; 260 261 /* This might end up being called from exit_mmap(), *before* the page 262 * tables are cleared. And __mmu_notifier_release() will delete us from 263 * the list of notifiers so that our invalidate_range() callback doesn't 264 * get called when the page tables are cleared. So we need to protect 265 * against hardware accessing those page tables. 266 * 267 * We do it by clearing the entry in the PASID table and then flushing 268 * the IOTLB and the PASID table caches. This might upset hardware; 269 * perhaps we'll want to point the PASID to a dummy PGD (like the zero 270 * page) so that we end up taking a fault that the hardware really 271 * *has* to handle gracefully without affecting other processes. 272 */ 273 rcu_read_lock(); 274 list_for_each_entry_rcu(sdev, &svm->devs, list) 275 intel_pasid_tear_down_entry(sdev->iommu, sdev->dev, 276 svm->pasid, true); 277 rcu_read_unlock(); 278 279 } 280 281 static const struct mmu_notifier_ops intel_mmuops = { 282 .release = intel_mm_release, 283 .arch_invalidate_secondary_tlbs = intel_arch_invalidate_secondary_tlbs, 284 }; 285 286 static int pasid_to_svm_sdev(struct device *dev, unsigned int pasid, 287 struct intel_svm **rsvm, 288 struct intel_svm_dev **rsdev) 289 { 290 struct intel_svm_dev *sdev = NULL; 291 struct intel_svm *svm; 292 293 if (pasid == IOMMU_PASID_INVALID || pasid >= PASID_MAX) 294 return -EINVAL; 295 296 svm = pasid_private_find(pasid); 297 if (IS_ERR(svm)) 298 return PTR_ERR(svm); 299 300 if (!svm) 301 goto out; 302 303 /* 304 * If we found svm for the PASID, there must be at least one device 305 * bond. 306 */ 307 if (WARN_ON(list_empty(&svm->devs))) 308 return -EINVAL; 309 sdev = svm_lookup_device_by_dev(svm, dev); 310 311 out: 312 *rsvm = svm; 313 *rsdev = sdev; 314 315 return 0; 316 } 317 318 static int intel_svm_bind_mm(struct intel_iommu *iommu, struct device *dev, 319 struct iommu_domain *domain, ioasid_t pasid) 320 { 321 struct device_domain_info *info = dev_iommu_priv_get(dev); 322 struct mm_struct *mm = domain->mm; 323 struct intel_svm_dev *sdev; 324 struct intel_svm *svm; 325 unsigned long sflags; 326 int ret = 0; 327 328 svm = pasid_private_find(pasid); 329 if (!svm) { 330 svm = kzalloc(sizeof(*svm), GFP_KERNEL); 331 if (!svm) 332 return -ENOMEM; 333 334 svm->pasid = pasid; 335 svm->mm = mm; 336 INIT_LIST_HEAD_RCU(&svm->devs); 337 338 svm->notifier.ops = &intel_mmuops; 339 ret = mmu_notifier_register(&svm->notifier, mm); 340 if (ret) { 341 kfree(svm); 342 return ret; 343 } 344 345 ret = pasid_private_add(svm->pasid, svm); 346 if (ret) { 347 mmu_notifier_unregister(&svm->notifier, mm); 348 kfree(svm); 349 return ret; 350 } 351 } 352 353 sdev = kzalloc(sizeof(*sdev), GFP_KERNEL); 354 if (!sdev) { 355 ret = -ENOMEM; 356 goto free_svm; 357 } 358 359 sdev->dev = dev; 360 sdev->iommu = iommu; 361 sdev->did = FLPT_DEFAULT_DID; 362 sdev->sid = PCI_DEVID(info->bus, info->devfn); 363 init_rcu_head(&sdev->rcu); 364 if (info->ats_enabled) { 365 sdev->qdep = info->ats_qdep; 366 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS) 367 sdev->qdep = 0; 368 } 369 370 /* Setup the pasid table: */ 371 sflags = cpu_feature_enabled(X86_FEATURE_LA57) ? PASID_FLAG_FL5LP : 0; 372 ret = intel_pasid_setup_first_level(iommu, dev, mm->pgd, pasid, 373 FLPT_DEFAULT_DID, sflags); 374 if (ret) 375 goto free_sdev; 376 377 list_add_rcu(&sdev->list, &svm->devs); 378 379 return 0; 380 381 free_sdev: 382 kfree(sdev); 383 free_svm: 384 if (list_empty(&svm->devs)) { 385 mmu_notifier_unregister(&svm->notifier, mm); 386 pasid_private_remove(pasid); 387 kfree(svm); 388 } 389 390 return ret; 391 } 392 393 void intel_svm_remove_dev_pasid(struct device *dev, u32 pasid) 394 { 395 struct intel_svm_dev *sdev; 396 struct intel_iommu *iommu; 397 struct intel_svm *svm; 398 struct mm_struct *mm; 399 400 iommu = device_to_iommu(dev, NULL, NULL); 401 if (!iommu) 402 return; 403 404 if (pasid_to_svm_sdev(dev, pasid, &svm, &sdev)) 405 return; 406 mm = svm->mm; 407 408 if (sdev) { 409 list_del_rcu(&sdev->list); 410 kfree_rcu(sdev, rcu); 411 412 if (list_empty(&svm->devs)) { 413 if (svm->notifier.ops) 414 mmu_notifier_unregister(&svm->notifier, mm); 415 pasid_private_remove(svm->pasid); 416 /* 417 * We mandate that no page faults may be outstanding 418 * for the PASID when intel_svm_unbind_mm() is called. 419 * If that is not obeyed, subtle errors will happen. 420 * Let's make them less subtle... 421 */ 422 memset(svm, 0x6b, sizeof(*svm)); 423 kfree(svm); 424 } 425 } 426 } 427 428 /* Page request queue descriptor */ 429 struct page_req_dsc { 430 union { 431 struct { 432 u64 type:8; 433 u64 pasid_present:1; 434 u64 priv_data_present:1; 435 u64 rsvd:6; 436 u64 rid:16; 437 u64 pasid:20; 438 u64 exe_req:1; 439 u64 pm_req:1; 440 u64 rsvd2:10; 441 }; 442 u64 qw_0; 443 }; 444 union { 445 struct { 446 u64 rd_req:1; 447 u64 wr_req:1; 448 u64 lpig:1; 449 u64 prg_index:9; 450 u64 addr:52; 451 }; 452 u64 qw_1; 453 }; 454 u64 priv_data[2]; 455 }; 456 457 static bool is_canonical_address(u64 addr) 458 { 459 int shift = 64 - (__VIRTUAL_MASK_SHIFT + 1); 460 long saddr = (long) addr; 461 462 return (((saddr << shift) >> shift) == saddr); 463 } 464 465 /** 466 * intel_drain_pasid_prq - Drain page requests and responses for a pasid 467 * @dev: target device 468 * @pasid: pasid for draining 469 * 470 * Drain all pending page requests and responses related to @pasid in both 471 * software and hardware. This is supposed to be called after the device 472 * driver has stopped DMA, the pasid entry has been cleared, and both IOTLB 473 * and DevTLB have been invalidated. 474 * 475 * It waits until all pending page requests for @pasid in the page fault 476 * queue are completed by the prq handling thread. Then follow the steps 477 * described in VT-d spec CH7.10 to drain all page requests and page 478 * responses pending in the hardware. 479 */ 480 void intel_drain_pasid_prq(struct device *dev, u32 pasid) 481 { 482 struct device_domain_info *info; 483 struct dmar_domain *domain; 484 struct intel_iommu *iommu; 485 struct qi_desc desc[3]; 486 struct pci_dev *pdev; 487 int head, tail; 488 u16 sid, did; 489 int qdep; 490 491 info = dev_iommu_priv_get(dev); 492 if (WARN_ON(!info || !dev_is_pci(dev))) 493 return; 494 495 if (!info->pri_enabled) 496 return; 497 498 iommu = info->iommu; 499 domain = info->domain; 500 pdev = to_pci_dev(dev); 501 sid = PCI_DEVID(info->bus, info->devfn); 502 did = domain_id_iommu(domain, iommu); 503 qdep = pci_ats_queue_depth(pdev); 504 505 /* 506 * Check and wait until all pending page requests in the queue are 507 * handled by the prq handling thread. 508 */ 509 prq_retry: 510 reinit_completion(&iommu->prq_complete); 511 tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; 512 head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; 513 while (head != tail) { 514 struct page_req_dsc *req; 515 516 req = &iommu->prq[head / sizeof(*req)]; 517 if (!req->pasid_present || req->pasid != pasid) { 518 head = (head + sizeof(*req)) & PRQ_RING_MASK; 519 continue; 520 } 521 522 wait_for_completion(&iommu->prq_complete); 523 goto prq_retry; 524 } 525 526 iopf_queue_flush_dev(dev); 527 528 /* 529 * Perform steps described in VT-d spec CH7.10 to drain page 530 * requests and responses in hardware. 531 */ 532 memset(desc, 0, sizeof(desc)); 533 desc[0].qw0 = QI_IWD_STATUS_DATA(QI_DONE) | 534 QI_IWD_FENCE | 535 QI_IWD_TYPE; 536 desc[1].qw0 = QI_EIOTLB_PASID(pasid) | 537 QI_EIOTLB_DID(did) | 538 QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) | 539 QI_EIOTLB_TYPE; 540 desc[2].qw0 = QI_DEV_EIOTLB_PASID(pasid) | 541 QI_DEV_EIOTLB_SID(sid) | 542 QI_DEV_EIOTLB_QDEP(qdep) | 543 QI_DEIOTLB_TYPE | 544 QI_DEV_IOTLB_PFSID(info->pfsid); 545 qi_retry: 546 reinit_completion(&iommu->prq_complete); 547 qi_submit_sync(iommu, desc, 3, QI_OPT_WAIT_DRAIN); 548 if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO) { 549 wait_for_completion(&iommu->prq_complete); 550 goto qi_retry; 551 } 552 } 553 554 static int prq_to_iommu_prot(struct page_req_dsc *req) 555 { 556 int prot = 0; 557 558 if (req->rd_req) 559 prot |= IOMMU_FAULT_PERM_READ; 560 if (req->wr_req) 561 prot |= IOMMU_FAULT_PERM_WRITE; 562 if (req->exe_req) 563 prot |= IOMMU_FAULT_PERM_EXEC; 564 if (req->pm_req) 565 prot |= IOMMU_FAULT_PERM_PRIV; 566 567 return prot; 568 } 569 570 static int intel_svm_prq_report(struct intel_iommu *iommu, struct device *dev, 571 struct page_req_dsc *desc) 572 { 573 struct iommu_fault_event event; 574 575 if (!dev || !dev_is_pci(dev)) 576 return -ENODEV; 577 578 /* Fill in event data for device specific processing */ 579 memset(&event, 0, sizeof(struct iommu_fault_event)); 580 event.fault.type = IOMMU_FAULT_PAGE_REQ; 581 event.fault.prm.addr = (u64)desc->addr << VTD_PAGE_SHIFT; 582 event.fault.prm.pasid = desc->pasid; 583 event.fault.prm.grpid = desc->prg_index; 584 event.fault.prm.perm = prq_to_iommu_prot(desc); 585 586 if (desc->lpig) 587 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; 588 if (desc->pasid_present) { 589 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; 590 event.fault.prm.flags |= IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID; 591 } 592 if (desc->priv_data_present) { 593 /* 594 * Set last page in group bit if private data is present, 595 * page response is required as it does for LPIG. 596 * iommu_report_device_fault() doesn't understand this vendor 597 * specific requirement thus we set last_page as a workaround. 598 */ 599 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; 600 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA; 601 event.fault.prm.private_data[0] = desc->priv_data[0]; 602 event.fault.prm.private_data[1] = desc->priv_data[1]; 603 } else if (dmar_latency_enabled(iommu, DMAR_LATENCY_PRQ)) { 604 /* 605 * If the private data fields are not used by hardware, use it 606 * to monitor the prq handle latency. 607 */ 608 event.fault.prm.private_data[0] = ktime_to_ns(ktime_get()); 609 } 610 611 return iommu_report_device_fault(dev, &event); 612 } 613 614 static void handle_bad_prq_event(struct intel_iommu *iommu, 615 struct page_req_dsc *req, int result) 616 { 617 struct qi_desc desc; 618 619 pr_err("%s: Invalid page request: %08llx %08llx\n", 620 iommu->name, ((unsigned long long *)req)[0], 621 ((unsigned long long *)req)[1]); 622 623 /* 624 * Per VT-d spec. v3.0 ch7.7, system software must 625 * respond with page group response if private data 626 * is present (PDP) or last page in group (LPIG) bit 627 * is set. This is an additional VT-d feature beyond 628 * PCI ATS spec. 629 */ 630 if (!req->lpig && !req->priv_data_present) 631 return; 632 633 desc.qw0 = QI_PGRP_PASID(req->pasid) | 634 QI_PGRP_DID(req->rid) | 635 QI_PGRP_PASID_P(req->pasid_present) | 636 QI_PGRP_PDP(req->priv_data_present) | 637 QI_PGRP_RESP_CODE(result) | 638 QI_PGRP_RESP_TYPE; 639 desc.qw1 = QI_PGRP_IDX(req->prg_index) | 640 QI_PGRP_LPIG(req->lpig); 641 642 if (req->priv_data_present) { 643 desc.qw2 = req->priv_data[0]; 644 desc.qw3 = req->priv_data[1]; 645 } else { 646 desc.qw2 = 0; 647 desc.qw3 = 0; 648 } 649 650 qi_submit_sync(iommu, &desc, 1, 0); 651 } 652 653 static irqreturn_t prq_event_thread(int irq, void *d) 654 { 655 struct intel_iommu *iommu = d; 656 struct page_req_dsc *req; 657 int head, tail, handled; 658 struct pci_dev *pdev; 659 u64 address; 660 661 /* 662 * Clear PPR bit before reading head/tail registers, to ensure that 663 * we get a new interrupt if needed. 664 */ 665 writel(DMA_PRS_PPR, iommu->reg + DMAR_PRS_REG); 666 667 tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; 668 head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; 669 handled = (head != tail); 670 while (head != tail) { 671 req = &iommu->prq[head / sizeof(*req)]; 672 address = (u64)req->addr << VTD_PAGE_SHIFT; 673 674 if (unlikely(!req->pasid_present)) { 675 pr_err("IOMMU: %s: Page request without PASID\n", 676 iommu->name); 677 bad_req: 678 handle_bad_prq_event(iommu, req, QI_RESP_INVALID); 679 goto prq_advance; 680 } 681 682 if (unlikely(!is_canonical_address(address))) { 683 pr_err("IOMMU: %s: Address is not canonical\n", 684 iommu->name); 685 goto bad_req; 686 } 687 688 if (unlikely(req->pm_req && (req->rd_req | req->wr_req))) { 689 pr_err("IOMMU: %s: Page request in Privilege Mode\n", 690 iommu->name); 691 goto bad_req; 692 } 693 694 if (unlikely(req->exe_req && req->rd_req)) { 695 pr_err("IOMMU: %s: Execution request not supported\n", 696 iommu->name); 697 goto bad_req; 698 } 699 700 /* Drop Stop Marker message. No need for a response. */ 701 if (unlikely(req->lpig && !req->rd_req && !req->wr_req)) 702 goto prq_advance; 703 704 pdev = pci_get_domain_bus_and_slot(iommu->segment, 705 PCI_BUS_NUM(req->rid), 706 req->rid & 0xff); 707 /* 708 * If prq is to be handled outside iommu driver via receiver of 709 * the fault notifiers, we skip the page response here. 710 */ 711 if (!pdev) 712 goto bad_req; 713 714 if (intel_svm_prq_report(iommu, &pdev->dev, req)) 715 handle_bad_prq_event(iommu, req, QI_RESP_INVALID); 716 else 717 trace_prq_report(iommu, &pdev->dev, req->qw_0, req->qw_1, 718 req->priv_data[0], req->priv_data[1], 719 iommu->prq_seq_number++); 720 pci_dev_put(pdev); 721 prq_advance: 722 head = (head + sizeof(*req)) & PRQ_RING_MASK; 723 } 724 725 dmar_writeq(iommu->reg + DMAR_PQH_REG, tail); 726 727 /* 728 * Clear the page request overflow bit and wake up all threads that 729 * are waiting for the completion of this handling. 730 */ 731 if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO) { 732 pr_info_ratelimited("IOMMU: %s: PRQ overflow detected\n", 733 iommu->name); 734 head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; 735 tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; 736 if (head == tail) { 737 iopf_queue_discard_partial(iommu->iopf_queue); 738 writel(DMA_PRS_PRO, iommu->reg + DMAR_PRS_REG); 739 pr_info_ratelimited("IOMMU: %s: PRQ overflow cleared", 740 iommu->name); 741 } 742 } 743 744 if (!completion_done(&iommu->prq_complete)) 745 complete(&iommu->prq_complete); 746 747 return IRQ_RETVAL(handled); 748 } 749 750 int intel_svm_page_response(struct device *dev, 751 struct iommu_fault_event *evt, 752 struct iommu_page_response *msg) 753 { 754 struct iommu_fault_page_request *prm; 755 struct intel_iommu *iommu; 756 bool private_present; 757 bool pasid_present; 758 bool last_page; 759 u8 bus, devfn; 760 int ret = 0; 761 u16 sid; 762 763 if (!dev || !dev_is_pci(dev)) 764 return -ENODEV; 765 766 iommu = device_to_iommu(dev, &bus, &devfn); 767 if (!iommu) 768 return -ENODEV; 769 770 if (!msg || !evt) 771 return -EINVAL; 772 773 prm = &evt->fault.prm; 774 sid = PCI_DEVID(bus, devfn); 775 pasid_present = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; 776 private_present = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA; 777 last_page = prm->flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; 778 779 if (!pasid_present) { 780 ret = -EINVAL; 781 goto out; 782 } 783 784 if (prm->pasid == 0 || prm->pasid >= PASID_MAX) { 785 ret = -EINVAL; 786 goto out; 787 } 788 789 /* 790 * Per VT-d spec. v3.0 ch7.7, system software must respond 791 * with page group response if private data is present (PDP) 792 * or last page in group (LPIG) bit is set. This is an 793 * additional VT-d requirement beyond PCI ATS spec. 794 */ 795 if (last_page || private_present) { 796 struct qi_desc desc; 797 798 desc.qw0 = QI_PGRP_PASID(prm->pasid) | QI_PGRP_DID(sid) | 799 QI_PGRP_PASID_P(pasid_present) | 800 QI_PGRP_PDP(private_present) | 801 QI_PGRP_RESP_CODE(msg->code) | 802 QI_PGRP_RESP_TYPE; 803 desc.qw1 = QI_PGRP_IDX(prm->grpid) | QI_PGRP_LPIG(last_page); 804 desc.qw2 = 0; 805 desc.qw3 = 0; 806 807 if (private_present) { 808 desc.qw2 = prm->private_data[0]; 809 desc.qw3 = prm->private_data[1]; 810 } else if (prm->private_data[0]) { 811 dmar_latency_update(iommu, DMAR_LATENCY_PRQ, 812 ktime_to_ns(ktime_get()) - prm->private_data[0]); 813 } 814 815 qi_submit_sync(iommu, &desc, 1, 0); 816 } 817 out: 818 return ret; 819 } 820 821 static int intel_svm_set_dev_pasid(struct iommu_domain *domain, 822 struct device *dev, ioasid_t pasid) 823 { 824 struct device_domain_info *info = dev_iommu_priv_get(dev); 825 struct intel_iommu *iommu = info->iommu; 826 827 return intel_svm_bind_mm(iommu, dev, domain, pasid); 828 } 829 830 static void intel_svm_domain_free(struct iommu_domain *domain) 831 { 832 kfree(to_dmar_domain(domain)); 833 } 834 835 static const struct iommu_domain_ops intel_svm_domain_ops = { 836 .set_dev_pasid = intel_svm_set_dev_pasid, 837 .free = intel_svm_domain_free 838 }; 839 840 struct iommu_domain *intel_svm_domain_alloc(void) 841 { 842 struct dmar_domain *domain; 843 844 domain = kzalloc(sizeof(*domain), GFP_KERNEL); 845 if (!domain) 846 return NULL; 847 domain->domain.ops = &intel_svm_domain_ops; 848 849 return &domain->domain; 850 } 851