1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc. 4 * Author: Joerg Roedel <[email protected]> 5 */ 6 7 #ifndef __LINUX_IOMMU_H 8 #define __LINUX_IOMMU_H 9 10 #include <linux/scatterlist.h> 11 #include <linux/device.h> 12 #include <linux/types.h> 13 #include <linux/errno.h> 14 #include <linux/err.h> 15 #include <linux/of.h> 16 #include <linux/ioasid.h> 17 #include <uapi/linux/iommu.h> 18 19 #define IOMMU_READ (1 << 0) 20 #define IOMMU_WRITE (1 << 1) 21 #define IOMMU_CACHE (1 << 2) /* DMA cache coherency */ 22 #define IOMMU_NOEXEC (1 << 3) 23 #define IOMMU_MMIO (1 << 4) /* e.g. things like MSI doorbells */ 24 /* 25 * Where the bus hardware includes a privilege level as part of its access type 26 * markings, and certain devices are capable of issuing transactions marked as 27 * either 'supervisor' or 'user', the IOMMU_PRIV flag requests that the other 28 * given permission flags only apply to accesses at the higher privilege level, 29 * and that unprivileged transactions should have as little access as possible. 30 * This would usually imply the same permissions as kernel mappings on the CPU, 31 * if the IOMMU page table format is equivalent. 32 */ 33 #define IOMMU_PRIV (1 << 5) 34 /* 35 * Non-coherent masters can use this page protection flag to set cacheable 36 * memory attributes for only a transparent outer level of cache, also known as 37 * the last-level or system cache. 38 */ 39 #define IOMMU_SYS_CACHE_ONLY (1 << 6) 40 41 struct iommu_ops; 42 struct iommu_group; 43 struct bus_type; 44 struct device; 45 struct iommu_domain; 46 struct notifier_block; 47 struct iommu_sva; 48 struct iommu_fault_event; 49 50 /* iommu fault flags */ 51 #define IOMMU_FAULT_READ 0x0 52 #define IOMMU_FAULT_WRITE 0x1 53 54 typedef int (*iommu_fault_handler_t)(struct iommu_domain *, 55 struct device *, unsigned long, int, void *); 56 typedef int (*iommu_mm_exit_handler_t)(struct device *dev, struct iommu_sva *, 57 void *); 58 typedef int (*iommu_dev_fault_handler_t)(struct iommu_fault *, void *); 59 60 struct iommu_domain_geometry { 61 dma_addr_t aperture_start; /* First address that can be mapped */ 62 dma_addr_t aperture_end; /* Last address that can be mapped */ 63 bool force_aperture; /* DMA only allowed in mappable range? */ 64 }; 65 66 /* Domain feature flags */ 67 #define __IOMMU_DOMAIN_PAGING (1U << 0) /* Support for iommu_map/unmap */ 68 #define __IOMMU_DOMAIN_DMA_API (1U << 1) /* Domain for use in DMA-API 69 implementation */ 70 #define __IOMMU_DOMAIN_PT (1U << 2) /* Domain is identity mapped */ 71 72 /* 73 * This are the possible domain-types 74 * 75 * IOMMU_DOMAIN_BLOCKED - All DMA is blocked, can be used to isolate 76 * devices 77 * IOMMU_DOMAIN_IDENTITY - DMA addresses are system physical addresses 78 * IOMMU_DOMAIN_UNMANAGED - DMA mappings managed by IOMMU-API user, used 79 * for VMs 80 * IOMMU_DOMAIN_DMA - Internally used for DMA-API implementations. 81 * This flag allows IOMMU drivers to implement 82 * certain optimizations for these domains 83 */ 84 #define IOMMU_DOMAIN_BLOCKED (0U) 85 #define IOMMU_DOMAIN_IDENTITY (__IOMMU_DOMAIN_PT) 86 #define IOMMU_DOMAIN_UNMANAGED (__IOMMU_DOMAIN_PAGING) 87 #define IOMMU_DOMAIN_DMA (__IOMMU_DOMAIN_PAGING | \ 88 __IOMMU_DOMAIN_DMA_API) 89 90 struct iommu_domain { 91 unsigned type; 92 const struct iommu_ops *ops; 93 unsigned long pgsize_bitmap; /* Bitmap of page sizes in use */ 94 iommu_fault_handler_t handler; 95 void *handler_token; 96 struct iommu_domain_geometry geometry; 97 void *iova_cookie; 98 }; 99 100 enum iommu_cap { 101 IOMMU_CAP_CACHE_COHERENCY, /* IOMMU can enforce cache coherent DMA 102 transactions */ 103 IOMMU_CAP_INTR_REMAP, /* IOMMU supports interrupt isolation */ 104 IOMMU_CAP_NOEXEC, /* IOMMU_NOEXEC flag */ 105 }; 106 107 /* 108 * Following constraints are specifc to FSL_PAMUV1: 109 * -aperture must be power of 2, and naturally aligned 110 * -number of windows must be power of 2, and address space size 111 * of each window is determined by aperture size / # of windows 112 * -the actual size of the mapped region of a window must be power 113 * of 2 starting with 4KB and physical address must be naturally 114 * aligned. 115 * DOMAIN_ATTR_FSL_PAMUV1 corresponds to the above mentioned contraints. 116 * The caller can invoke iommu_domain_get_attr to check if the underlying 117 * iommu implementation supports these constraints. 118 */ 119 120 enum iommu_attr { 121 DOMAIN_ATTR_GEOMETRY, 122 DOMAIN_ATTR_PAGING, 123 DOMAIN_ATTR_WINDOWS, 124 DOMAIN_ATTR_FSL_PAMU_STASH, 125 DOMAIN_ATTR_FSL_PAMU_ENABLE, 126 DOMAIN_ATTR_FSL_PAMUV1, 127 DOMAIN_ATTR_NESTING, /* two stages of translation */ 128 DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE, 129 DOMAIN_ATTR_MAX, 130 }; 131 132 /* These are the possible reserved region types */ 133 enum iommu_resv_type { 134 /* Memory regions which must be mapped 1:1 at all times */ 135 IOMMU_RESV_DIRECT, 136 /* 137 * Memory regions which are advertised to be 1:1 but are 138 * commonly considered relaxable in some conditions, 139 * for instance in device assignment use case (USB, Graphics) 140 */ 141 IOMMU_RESV_DIRECT_RELAXABLE, 142 /* Arbitrary "never map this or give it to a device" address ranges */ 143 IOMMU_RESV_RESERVED, 144 /* Hardware MSI region (untranslated) */ 145 IOMMU_RESV_MSI, 146 /* Software-managed MSI translation window */ 147 IOMMU_RESV_SW_MSI, 148 }; 149 150 /** 151 * struct iommu_resv_region - descriptor for a reserved memory region 152 * @list: Linked list pointers 153 * @start: System physical start address of the region 154 * @length: Length of the region in bytes 155 * @prot: IOMMU Protection flags (READ/WRITE/...) 156 * @type: Type of the reserved region 157 */ 158 struct iommu_resv_region { 159 struct list_head list; 160 phys_addr_t start; 161 size_t length; 162 int prot; 163 enum iommu_resv_type type; 164 }; 165 166 /* Per device IOMMU features */ 167 enum iommu_dev_features { 168 IOMMU_DEV_FEAT_AUX, /* Aux-domain feature */ 169 IOMMU_DEV_FEAT_SVA, /* Shared Virtual Addresses */ 170 }; 171 172 #define IOMMU_PASID_INVALID (-1U) 173 174 /** 175 * struct iommu_sva_ops - device driver callbacks for an SVA context 176 * 177 * @mm_exit: called when the mm is about to be torn down by exit_mmap. After 178 * @mm_exit returns, the device must not issue any more transaction 179 * with the PASID given as argument. 180 * 181 * The @mm_exit handler is allowed to sleep. Be careful about the 182 * locks taken in @mm_exit, because they might lead to deadlocks if 183 * they are also held when dropping references to the mm. Consider the 184 * following call chain: 185 * mutex_lock(A); mmput(mm) -> exit_mm() -> @mm_exit() -> mutex_lock(A) 186 * Using mmput_async() prevents this scenario. 187 * 188 */ 189 struct iommu_sva_ops { 190 iommu_mm_exit_handler_t mm_exit; 191 }; 192 193 #ifdef CONFIG_IOMMU_API 194 195 /** 196 * struct iommu_iotlb_gather - Range information for a pending IOTLB flush 197 * 198 * @start: IOVA representing the start of the range to be flushed 199 * @end: IOVA representing the end of the range to be flushed (exclusive) 200 * @pgsize: The interval at which to perform the flush 201 * 202 * This structure is intended to be updated by multiple calls to the 203 * ->unmap() function in struct iommu_ops before eventually being passed 204 * into ->iotlb_sync(). 205 */ 206 struct iommu_iotlb_gather { 207 unsigned long start; 208 unsigned long end; 209 size_t pgsize; 210 }; 211 212 /** 213 * struct iommu_ops - iommu ops and capabilities 214 * @capable: check capability 215 * @domain_alloc: allocate iommu domain 216 * @domain_free: free iommu domain 217 * @attach_dev: attach device to an iommu domain 218 * @detach_dev: detach device from an iommu domain 219 * @map: map a physically contiguous memory region to an iommu domain 220 * @unmap: unmap a physically contiguous memory region from an iommu domain 221 * @flush_iotlb_all: Synchronously flush all hardware TLBs for this domain 222 * @iotlb_sync_map: Sync mappings created recently using @map to the hardware 223 * @iotlb_sync: Flush all queued ranges from the hardware TLBs and empty flush 224 * queue 225 * @iova_to_phys: translate iova to physical address 226 * @add_device: add device to iommu grouping 227 * @remove_device: remove device from iommu grouping 228 * @device_group: find iommu group for a particular device 229 * @domain_get_attr: Query domain attributes 230 * @domain_set_attr: Change domain attributes 231 * @get_resv_regions: Request list of reserved regions for a device 232 * @put_resv_regions: Free list of reserved regions for a device 233 * @apply_resv_region: Temporary helper call-back for iova reserved ranges 234 * @domain_window_enable: Configure and enable a particular window for a domain 235 * @domain_window_disable: Disable a particular window for a domain 236 * @of_xlate: add OF master IDs to iommu grouping 237 * @is_attach_deferred: Check if domain attach should be deferred from iommu 238 * driver init to device driver init (default no) 239 * @dev_has/enable/disable_feat: per device entries to check/enable/disable 240 * iommu specific features. 241 * @dev_feat_enabled: check enabled feature 242 * @aux_attach/detach_dev: aux-domain specific attach/detach entries. 243 * @aux_get_pasid: get the pasid given an aux-domain 244 * @sva_bind: Bind process address space to device 245 * @sva_unbind: Unbind process address space from device 246 * @sva_get_pasid: Get PASID associated to a SVA handle 247 * @page_response: handle page request response 248 * @cache_invalidate: invalidate translation caches 249 * @sva_bind_gpasid: bind guest pasid and mm 250 * @sva_unbind_gpasid: unbind guest pasid and mm 251 * @pgsize_bitmap: bitmap of all possible supported page sizes 252 * @owner: Driver module providing these ops 253 */ 254 struct iommu_ops { 255 bool (*capable)(enum iommu_cap); 256 257 /* Domain allocation and freeing by the iommu driver */ 258 struct iommu_domain *(*domain_alloc)(unsigned iommu_domain_type); 259 void (*domain_free)(struct iommu_domain *); 260 261 int (*attach_dev)(struct iommu_domain *domain, struct device *dev); 262 void (*detach_dev)(struct iommu_domain *domain, struct device *dev); 263 int (*map)(struct iommu_domain *domain, unsigned long iova, 264 phys_addr_t paddr, size_t size, int prot, gfp_t gfp); 265 size_t (*unmap)(struct iommu_domain *domain, unsigned long iova, 266 size_t size, struct iommu_iotlb_gather *iotlb_gather); 267 void (*flush_iotlb_all)(struct iommu_domain *domain); 268 void (*iotlb_sync_map)(struct iommu_domain *domain); 269 void (*iotlb_sync)(struct iommu_domain *domain, 270 struct iommu_iotlb_gather *iotlb_gather); 271 phys_addr_t (*iova_to_phys)(struct iommu_domain *domain, dma_addr_t iova); 272 int (*add_device)(struct device *dev); 273 void (*remove_device)(struct device *dev); 274 struct iommu_group *(*device_group)(struct device *dev); 275 int (*domain_get_attr)(struct iommu_domain *domain, 276 enum iommu_attr attr, void *data); 277 int (*domain_set_attr)(struct iommu_domain *domain, 278 enum iommu_attr attr, void *data); 279 280 /* Request/Free a list of reserved regions for a device */ 281 void (*get_resv_regions)(struct device *dev, struct list_head *list); 282 void (*put_resv_regions)(struct device *dev, struct list_head *list); 283 void (*apply_resv_region)(struct device *dev, 284 struct iommu_domain *domain, 285 struct iommu_resv_region *region); 286 287 /* Window handling functions */ 288 int (*domain_window_enable)(struct iommu_domain *domain, u32 wnd_nr, 289 phys_addr_t paddr, u64 size, int prot); 290 void (*domain_window_disable)(struct iommu_domain *domain, u32 wnd_nr); 291 292 int (*of_xlate)(struct device *dev, struct of_phandle_args *args); 293 bool (*is_attach_deferred)(struct iommu_domain *domain, struct device *dev); 294 295 /* Per device IOMMU features */ 296 bool (*dev_has_feat)(struct device *dev, enum iommu_dev_features f); 297 bool (*dev_feat_enabled)(struct device *dev, enum iommu_dev_features f); 298 int (*dev_enable_feat)(struct device *dev, enum iommu_dev_features f); 299 int (*dev_disable_feat)(struct device *dev, enum iommu_dev_features f); 300 301 /* Aux-domain specific attach/detach entries */ 302 int (*aux_attach_dev)(struct iommu_domain *domain, struct device *dev); 303 void (*aux_detach_dev)(struct iommu_domain *domain, struct device *dev); 304 int (*aux_get_pasid)(struct iommu_domain *domain, struct device *dev); 305 306 struct iommu_sva *(*sva_bind)(struct device *dev, struct mm_struct *mm, 307 void *drvdata); 308 void (*sva_unbind)(struct iommu_sva *handle); 309 int (*sva_get_pasid)(struct iommu_sva *handle); 310 311 int (*page_response)(struct device *dev, 312 struct iommu_fault_event *evt, 313 struct iommu_page_response *msg); 314 int (*cache_invalidate)(struct iommu_domain *domain, struct device *dev, 315 struct iommu_cache_invalidate_info *inv_info); 316 int (*sva_bind_gpasid)(struct iommu_domain *domain, 317 struct device *dev, struct iommu_gpasid_bind_data *data); 318 319 int (*sva_unbind_gpasid)(struct device *dev, int pasid); 320 321 unsigned long pgsize_bitmap; 322 struct module *owner; 323 }; 324 325 /** 326 * struct iommu_device - IOMMU core representation of one IOMMU hardware 327 * instance 328 * @list: Used by the iommu-core to keep a list of registered iommus 329 * @ops: iommu-ops for talking to this iommu 330 * @dev: struct device for sysfs handling 331 */ 332 struct iommu_device { 333 struct list_head list; 334 const struct iommu_ops *ops; 335 struct fwnode_handle *fwnode; 336 struct device *dev; 337 }; 338 339 /** 340 * struct iommu_fault_event - Generic fault event 341 * 342 * Can represent recoverable faults such as a page requests or 343 * unrecoverable faults such as DMA or IRQ remapping faults. 344 * 345 * @fault: fault descriptor 346 * @list: pending fault event list, used for tracking responses 347 */ 348 struct iommu_fault_event { 349 struct iommu_fault fault; 350 struct list_head list; 351 }; 352 353 /** 354 * struct iommu_fault_param - per-device IOMMU fault data 355 * @handler: Callback function to handle IOMMU faults at device level 356 * @data: handler private data 357 * @faults: holds the pending faults which needs response 358 * @lock: protect pending faults list 359 */ 360 struct iommu_fault_param { 361 iommu_dev_fault_handler_t handler; 362 void *data; 363 struct list_head faults; 364 struct mutex lock; 365 }; 366 367 /** 368 * struct dev_iommu - Collection of per-device IOMMU data 369 * 370 * @fault_param: IOMMU detected device fault reporting data 371 * @fwspec: IOMMU fwspec data 372 * @priv: IOMMU Driver private data 373 * 374 * TODO: migrate other per device data pointers under iommu_dev_data, e.g. 375 * struct iommu_group *iommu_group; 376 */ 377 struct dev_iommu { 378 struct mutex lock; 379 struct iommu_fault_param *fault_param; 380 struct iommu_fwspec *fwspec; 381 void *priv; 382 }; 383 384 int iommu_device_register(struct iommu_device *iommu); 385 void iommu_device_unregister(struct iommu_device *iommu); 386 int iommu_device_sysfs_add(struct iommu_device *iommu, 387 struct device *parent, 388 const struct attribute_group **groups, 389 const char *fmt, ...) __printf(4, 5); 390 void iommu_device_sysfs_remove(struct iommu_device *iommu); 391 int iommu_device_link(struct iommu_device *iommu, struct device *link); 392 void iommu_device_unlink(struct iommu_device *iommu, struct device *link); 393 394 static inline void __iommu_device_set_ops(struct iommu_device *iommu, 395 const struct iommu_ops *ops) 396 { 397 iommu->ops = ops; 398 } 399 400 #define iommu_device_set_ops(iommu, ops) \ 401 do { \ 402 struct iommu_ops *__ops = (struct iommu_ops *)(ops); \ 403 __ops->owner = THIS_MODULE; \ 404 __iommu_device_set_ops(iommu, __ops); \ 405 } while (0) 406 407 static inline void iommu_device_set_fwnode(struct iommu_device *iommu, 408 struct fwnode_handle *fwnode) 409 { 410 iommu->fwnode = fwnode; 411 } 412 413 static inline struct iommu_device *dev_to_iommu_device(struct device *dev) 414 { 415 return (struct iommu_device *)dev_get_drvdata(dev); 416 } 417 418 static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather) 419 { 420 *gather = (struct iommu_iotlb_gather) { 421 .start = ULONG_MAX, 422 }; 423 } 424 425 #define IOMMU_GROUP_NOTIFY_ADD_DEVICE 1 /* Device added */ 426 #define IOMMU_GROUP_NOTIFY_DEL_DEVICE 2 /* Pre Device removed */ 427 #define IOMMU_GROUP_NOTIFY_BIND_DRIVER 3 /* Pre Driver bind */ 428 #define IOMMU_GROUP_NOTIFY_BOUND_DRIVER 4 /* Post Driver bind */ 429 #define IOMMU_GROUP_NOTIFY_UNBIND_DRIVER 5 /* Pre Driver unbind */ 430 #define IOMMU_GROUP_NOTIFY_UNBOUND_DRIVER 6 /* Post Driver unbind */ 431 432 extern int bus_set_iommu(struct bus_type *bus, const struct iommu_ops *ops); 433 extern bool iommu_present(struct bus_type *bus); 434 extern bool iommu_capable(struct bus_type *bus, enum iommu_cap cap); 435 extern struct iommu_domain *iommu_domain_alloc(struct bus_type *bus); 436 extern struct iommu_group *iommu_group_get_by_id(int id); 437 extern void iommu_domain_free(struct iommu_domain *domain); 438 extern int iommu_attach_device(struct iommu_domain *domain, 439 struct device *dev); 440 extern void iommu_detach_device(struct iommu_domain *domain, 441 struct device *dev); 442 extern int iommu_cache_invalidate(struct iommu_domain *domain, 443 struct device *dev, 444 struct iommu_cache_invalidate_info *inv_info); 445 extern int iommu_sva_bind_gpasid(struct iommu_domain *domain, 446 struct device *dev, struct iommu_gpasid_bind_data *data); 447 extern int iommu_sva_unbind_gpasid(struct iommu_domain *domain, 448 struct device *dev, ioasid_t pasid); 449 extern struct iommu_domain *iommu_get_domain_for_dev(struct device *dev); 450 extern struct iommu_domain *iommu_get_dma_domain(struct device *dev); 451 extern int iommu_map(struct iommu_domain *domain, unsigned long iova, 452 phys_addr_t paddr, size_t size, int prot); 453 extern int iommu_map_atomic(struct iommu_domain *domain, unsigned long iova, 454 phys_addr_t paddr, size_t size, int prot); 455 extern size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, 456 size_t size); 457 extern size_t iommu_unmap_fast(struct iommu_domain *domain, 458 unsigned long iova, size_t size, 459 struct iommu_iotlb_gather *iotlb_gather); 460 extern size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova, 461 struct scatterlist *sg,unsigned int nents, int prot); 462 extern size_t iommu_map_sg_atomic(struct iommu_domain *domain, 463 unsigned long iova, struct scatterlist *sg, 464 unsigned int nents, int prot); 465 extern phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova); 466 extern void iommu_set_fault_handler(struct iommu_domain *domain, 467 iommu_fault_handler_t handler, void *token); 468 469 extern void iommu_get_resv_regions(struct device *dev, struct list_head *list); 470 extern void iommu_put_resv_regions(struct device *dev, struct list_head *list); 471 extern void generic_iommu_put_resv_regions(struct device *dev, 472 struct list_head *list); 473 extern int iommu_request_dm_for_dev(struct device *dev); 474 extern int iommu_request_dma_domain_for_dev(struct device *dev); 475 extern void iommu_set_default_passthrough(bool cmd_line); 476 extern void iommu_set_default_translated(bool cmd_line); 477 extern bool iommu_default_passthrough(void); 478 extern struct iommu_resv_region * 479 iommu_alloc_resv_region(phys_addr_t start, size_t length, int prot, 480 enum iommu_resv_type type); 481 extern int iommu_get_group_resv_regions(struct iommu_group *group, 482 struct list_head *head); 483 484 extern int iommu_attach_group(struct iommu_domain *domain, 485 struct iommu_group *group); 486 extern void iommu_detach_group(struct iommu_domain *domain, 487 struct iommu_group *group); 488 extern struct iommu_group *iommu_group_alloc(void); 489 extern void *iommu_group_get_iommudata(struct iommu_group *group); 490 extern void iommu_group_set_iommudata(struct iommu_group *group, 491 void *iommu_data, 492 void (*release)(void *iommu_data)); 493 extern int iommu_group_set_name(struct iommu_group *group, const char *name); 494 extern int iommu_group_add_device(struct iommu_group *group, 495 struct device *dev); 496 extern void iommu_group_remove_device(struct device *dev); 497 extern int iommu_group_for_each_dev(struct iommu_group *group, void *data, 498 int (*fn)(struct device *, void *)); 499 extern struct iommu_group *iommu_group_get(struct device *dev); 500 extern struct iommu_group *iommu_group_ref_get(struct iommu_group *group); 501 extern void iommu_group_put(struct iommu_group *group); 502 extern int iommu_group_register_notifier(struct iommu_group *group, 503 struct notifier_block *nb); 504 extern int iommu_group_unregister_notifier(struct iommu_group *group, 505 struct notifier_block *nb); 506 extern int iommu_register_device_fault_handler(struct device *dev, 507 iommu_dev_fault_handler_t handler, 508 void *data); 509 510 extern int iommu_unregister_device_fault_handler(struct device *dev); 511 512 extern int iommu_report_device_fault(struct device *dev, 513 struct iommu_fault_event *evt); 514 extern int iommu_page_response(struct device *dev, 515 struct iommu_page_response *msg); 516 517 extern int iommu_group_id(struct iommu_group *group); 518 extern struct iommu_group *iommu_group_get_for_dev(struct device *dev); 519 extern struct iommu_domain *iommu_group_default_domain(struct iommu_group *); 520 521 extern int iommu_domain_get_attr(struct iommu_domain *domain, enum iommu_attr, 522 void *data); 523 extern int iommu_domain_set_attr(struct iommu_domain *domain, enum iommu_attr, 524 void *data); 525 526 /* Window handling function prototypes */ 527 extern int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr, 528 phys_addr_t offset, u64 size, 529 int prot); 530 extern void iommu_domain_window_disable(struct iommu_domain *domain, u32 wnd_nr); 531 532 extern int report_iommu_fault(struct iommu_domain *domain, struct device *dev, 533 unsigned long iova, int flags); 534 535 static inline void iommu_flush_tlb_all(struct iommu_domain *domain) 536 { 537 if (domain->ops->flush_iotlb_all) 538 domain->ops->flush_iotlb_all(domain); 539 } 540 541 static inline void iommu_tlb_sync(struct iommu_domain *domain, 542 struct iommu_iotlb_gather *iotlb_gather) 543 { 544 if (domain->ops->iotlb_sync) 545 domain->ops->iotlb_sync(domain, iotlb_gather); 546 547 iommu_iotlb_gather_init(iotlb_gather); 548 } 549 550 static inline void iommu_iotlb_gather_add_page(struct iommu_domain *domain, 551 struct iommu_iotlb_gather *gather, 552 unsigned long iova, size_t size) 553 { 554 unsigned long start = iova, end = start + size; 555 556 /* 557 * If the new page is disjoint from the current range or is mapped at 558 * a different granularity, then sync the TLB so that the gather 559 * structure can be rewritten. 560 */ 561 if (gather->pgsize != size || 562 end < gather->start || start > gather->end) { 563 if (gather->pgsize) 564 iommu_tlb_sync(domain, gather); 565 gather->pgsize = size; 566 } 567 568 if (gather->end < end) 569 gather->end = end; 570 571 if (gather->start > start) 572 gather->start = start; 573 } 574 575 /* PCI device grouping function */ 576 extern struct iommu_group *pci_device_group(struct device *dev); 577 /* Generic device grouping function */ 578 extern struct iommu_group *generic_device_group(struct device *dev); 579 /* FSL-MC device grouping function */ 580 struct iommu_group *fsl_mc_device_group(struct device *dev); 581 582 /** 583 * struct iommu_fwspec - per-device IOMMU instance data 584 * @ops: ops for this device's IOMMU 585 * @iommu_fwnode: firmware handle for this device's IOMMU 586 * @iommu_priv: IOMMU driver private data for this device 587 * @num_pasid_bits: number of PASID bits supported by this device 588 * @num_ids: number of associated device IDs 589 * @ids: IDs which this device may present to the IOMMU 590 */ 591 struct iommu_fwspec { 592 const struct iommu_ops *ops; 593 struct fwnode_handle *iommu_fwnode; 594 u32 flags; 595 u32 num_pasid_bits; 596 unsigned int num_ids; 597 u32 ids[]; 598 }; 599 600 /* ATS is supported */ 601 #define IOMMU_FWSPEC_PCI_RC_ATS (1 << 0) 602 603 /** 604 * struct iommu_sva - handle to a device-mm bond 605 */ 606 struct iommu_sva { 607 struct device *dev; 608 const struct iommu_sva_ops *ops; 609 }; 610 611 int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode, 612 const struct iommu_ops *ops); 613 void iommu_fwspec_free(struct device *dev); 614 int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids); 615 const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode); 616 617 static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev) 618 { 619 if (dev->iommu) 620 return dev->iommu->fwspec; 621 else 622 return NULL; 623 } 624 625 static inline void dev_iommu_fwspec_set(struct device *dev, 626 struct iommu_fwspec *fwspec) 627 { 628 dev->iommu->fwspec = fwspec; 629 } 630 631 static inline void *dev_iommu_priv_get(struct device *dev) 632 { 633 return dev->iommu->priv; 634 } 635 636 static inline void dev_iommu_priv_set(struct device *dev, void *priv) 637 { 638 dev->iommu->priv = priv; 639 } 640 641 int iommu_probe_device(struct device *dev); 642 void iommu_release_device(struct device *dev); 643 644 bool iommu_dev_has_feature(struct device *dev, enum iommu_dev_features f); 645 int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features f); 646 int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features f); 647 bool iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features f); 648 int iommu_aux_attach_device(struct iommu_domain *domain, struct device *dev); 649 void iommu_aux_detach_device(struct iommu_domain *domain, struct device *dev); 650 int iommu_aux_get_pasid(struct iommu_domain *domain, struct device *dev); 651 652 struct iommu_sva *iommu_sva_bind_device(struct device *dev, 653 struct mm_struct *mm, 654 void *drvdata); 655 void iommu_sva_unbind_device(struct iommu_sva *handle); 656 int iommu_sva_set_ops(struct iommu_sva *handle, 657 const struct iommu_sva_ops *ops); 658 int iommu_sva_get_pasid(struct iommu_sva *handle); 659 660 #else /* CONFIG_IOMMU_API */ 661 662 struct iommu_ops {}; 663 struct iommu_group {}; 664 struct iommu_fwspec {}; 665 struct iommu_device {}; 666 struct iommu_fault_param {}; 667 struct iommu_iotlb_gather {}; 668 669 static inline bool iommu_present(struct bus_type *bus) 670 { 671 return false; 672 } 673 674 static inline bool iommu_capable(struct bus_type *bus, enum iommu_cap cap) 675 { 676 return false; 677 } 678 679 static inline struct iommu_domain *iommu_domain_alloc(struct bus_type *bus) 680 { 681 return NULL; 682 } 683 684 static inline struct iommu_group *iommu_group_get_by_id(int id) 685 { 686 return NULL; 687 } 688 689 static inline void iommu_domain_free(struct iommu_domain *domain) 690 { 691 } 692 693 static inline int iommu_attach_device(struct iommu_domain *domain, 694 struct device *dev) 695 { 696 return -ENODEV; 697 } 698 699 static inline void iommu_detach_device(struct iommu_domain *domain, 700 struct device *dev) 701 { 702 } 703 704 static inline struct iommu_domain *iommu_get_domain_for_dev(struct device *dev) 705 { 706 return NULL; 707 } 708 709 static inline int iommu_map(struct iommu_domain *domain, unsigned long iova, 710 phys_addr_t paddr, size_t size, int prot) 711 { 712 return -ENODEV; 713 } 714 715 static inline int iommu_map_atomic(struct iommu_domain *domain, 716 unsigned long iova, phys_addr_t paddr, 717 size_t size, int prot) 718 { 719 return -ENODEV; 720 } 721 722 static inline size_t iommu_unmap(struct iommu_domain *domain, 723 unsigned long iova, size_t size) 724 { 725 return 0; 726 } 727 728 static inline size_t iommu_unmap_fast(struct iommu_domain *domain, 729 unsigned long iova, int gfp_order, 730 struct iommu_iotlb_gather *iotlb_gather) 731 { 732 return 0; 733 } 734 735 static inline size_t iommu_map_sg(struct iommu_domain *domain, 736 unsigned long iova, struct scatterlist *sg, 737 unsigned int nents, int prot) 738 { 739 return 0; 740 } 741 742 static inline size_t iommu_map_sg_atomic(struct iommu_domain *domain, 743 unsigned long iova, struct scatterlist *sg, 744 unsigned int nents, int prot) 745 { 746 return 0; 747 } 748 749 static inline void iommu_flush_tlb_all(struct iommu_domain *domain) 750 { 751 } 752 753 static inline void iommu_tlb_sync(struct iommu_domain *domain, 754 struct iommu_iotlb_gather *iotlb_gather) 755 { 756 } 757 758 static inline int iommu_domain_window_enable(struct iommu_domain *domain, 759 u32 wnd_nr, phys_addr_t paddr, 760 u64 size, int prot) 761 { 762 return -ENODEV; 763 } 764 765 static inline void iommu_domain_window_disable(struct iommu_domain *domain, 766 u32 wnd_nr) 767 { 768 } 769 770 static inline phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) 771 { 772 return 0; 773 } 774 775 static inline void iommu_set_fault_handler(struct iommu_domain *domain, 776 iommu_fault_handler_t handler, void *token) 777 { 778 } 779 780 static inline void iommu_get_resv_regions(struct device *dev, 781 struct list_head *list) 782 { 783 } 784 785 static inline void iommu_put_resv_regions(struct device *dev, 786 struct list_head *list) 787 { 788 } 789 790 static inline int iommu_get_group_resv_regions(struct iommu_group *group, 791 struct list_head *head) 792 { 793 return -ENODEV; 794 } 795 796 static inline int iommu_request_dm_for_dev(struct device *dev) 797 { 798 return -ENODEV; 799 } 800 801 static inline int iommu_request_dma_domain_for_dev(struct device *dev) 802 { 803 return -ENODEV; 804 } 805 806 static inline void iommu_set_default_passthrough(bool cmd_line) 807 { 808 } 809 810 static inline void iommu_set_default_translated(bool cmd_line) 811 { 812 } 813 814 static inline bool iommu_default_passthrough(void) 815 { 816 return true; 817 } 818 819 static inline int iommu_attach_group(struct iommu_domain *domain, 820 struct iommu_group *group) 821 { 822 return -ENODEV; 823 } 824 825 static inline void iommu_detach_group(struct iommu_domain *domain, 826 struct iommu_group *group) 827 { 828 } 829 830 static inline struct iommu_group *iommu_group_alloc(void) 831 { 832 return ERR_PTR(-ENODEV); 833 } 834 835 static inline void *iommu_group_get_iommudata(struct iommu_group *group) 836 { 837 return NULL; 838 } 839 840 static inline void iommu_group_set_iommudata(struct iommu_group *group, 841 void *iommu_data, 842 void (*release)(void *iommu_data)) 843 { 844 } 845 846 static inline int iommu_group_set_name(struct iommu_group *group, 847 const char *name) 848 { 849 return -ENODEV; 850 } 851 852 static inline int iommu_group_add_device(struct iommu_group *group, 853 struct device *dev) 854 { 855 return -ENODEV; 856 } 857 858 static inline void iommu_group_remove_device(struct device *dev) 859 { 860 } 861 862 static inline int iommu_group_for_each_dev(struct iommu_group *group, 863 void *data, 864 int (*fn)(struct device *, void *)) 865 { 866 return -ENODEV; 867 } 868 869 static inline struct iommu_group *iommu_group_get(struct device *dev) 870 { 871 return NULL; 872 } 873 874 static inline void iommu_group_put(struct iommu_group *group) 875 { 876 } 877 878 static inline int iommu_group_register_notifier(struct iommu_group *group, 879 struct notifier_block *nb) 880 { 881 return -ENODEV; 882 } 883 884 static inline int iommu_group_unregister_notifier(struct iommu_group *group, 885 struct notifier_block *nb) 886 { 887 return 0; 888 } 889 890 static inline 891 int iommu_register_device_fault_handler(struct device *dev, 892 iommu_dev_fault_handler_t handler, 893 void *data) 894 { 895 return -ENODEV; 896 } 897 898 static inline int iommu_unregister_device_fault_handler(struct device *dev) 899 { 900 return 0; 901 } 902 903 static inline 904 int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt) 905 { 906 return -ENODEV; 907 } 908 909 static inline int iommu_page_response(struct device *dev, 910 struct iommu_page_response *msg) 911 { 912 return -ENODEV; 913 } 914 915 static inline int iommu_group_id(struct iommu_group *group) 916 { 917 return -ENODEV; 918 } 919 920 static inline int iommu_domain_get_attr(struct iommu_domain *domain, 921 enum iommu_attr attr, void *data) 922 { 923 return -EINVAL; 924 } 925 926 static inline int iommu_domain_set_attr(struct iommu_domain *domain, 927 enum iommu_attr attr, void *data) 928 { 929 return -EINVAL; 930 } 931 932 static inline int iommu_device_register(struct iommu_device *iommu) 933 { 934 return -ENODEV; 935 } 936 937 static inline void iommu_device_set_ops(struct iommu_device *iommu, 938 const struct iommu_ops *ops) 939 { 940 } 941 942 static inline void iommu_device_set_fwnode(struct iommu_device *iommu, 943 struct fwnode_handle *fwnode) 944 { 945 } 946 947 static inline struct iommu_device *dev_to_iommu_device(struct device *dev) 948 { 949 return NULL; 950 } 951 952 static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather) 953 { 954 } 955 956 static inline void iommu_iotlb_gather_add_page(struct iommu_domain *domain, 957 struct iommu_iotlb_gather *gather, 958 unsigned long iova, size_t size) 959 { 960 } 961 962 static inline void iommu_device_unregister(struct iommu_device *iommu) 963 { 964 } 965 966 static inline int iommu_device_sysfs_add(struct iommu_device *iommu, 967 struct device *parent, 968 const struct attribute_group **groups, 969 const char *fmt, ...) 970 { 971 return -ENODEV; 972 } 973 974 static inline void iommu_device_sysfs_remove(struct iommu_device *iommu) 975 { 976 } 977 978 static inline int iommu_device_link(struct device *dev, struct device *link) 979 { 980 return -EINVAL; 981 } 982 983 static inline void iommu_device_unlink(struct device *dev, struct device *link) 984 { 985 } 986 987 static inline int iommu_fwspec_init(struct device *dev, 988 struct fwnode_handle *iommu_fwnode, 989 const struct iommu_ops *ops) 990 { 991 return -ENODEV; 992 } 993 994 static inline void iommu_fwspec_free(struct device *dev) 995 { 996 } 997 998 static inline int iommu_fwspec_add_ids(struct device *dev, u32 *ids, 999 int num_ids) 1000 { 1001 return -ENODEV; 1002 } 1003 1004 static inline 1005 const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode) 1006 { 1007 return NULL; 1008 } 1009 1010 static inline bool 1011 iommu_dev_has_feature(struct device *dev, enum iommu_dev_features feat) 1012 { 1013 return false; 1014 } 1015 1016 static inline bool 1017 iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features feat) 1018 { 1019 return false; 1020 } 1021 1022 static inline int 1023 iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features feat) 1024 { 1025 return -ENODEV; 1026 } 1027 1028 static inline int 1029 iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat) 1030 { 1031 return -ENODEV; 1032 } 1033 1034 static inline int 1035 iommu_aux_attach_device(struct iommu_domain *domain, struct device *dev) 1036 { 1037 return -ENODEV; 1038 } 1039 1040 static inline void 1041 iommu_aux_detach_device(struct iommu_domain *domain, struct device *dev) 1042 { 1043 } 1044 1045 static inline int 1046 iommu_aux_get_pasid(struct iommu_domain *domain, struct device *dev) 1047 { 1048 return -ENODEV; 1049 } 1050 1051 static inline struct iommu_sva * 1052 iommu_sva_bind_device(struct device *dev, struct mm_struct *mm, void *drvdata) 1053 { 1054 return NULL; 1055 } 1056 1057 static inline void iommu_sva_unbind_device(struct iommu_sva *handle) 1058 { 1059 } 1060 1061 static inline int iommu_sva_set_ops(struct iommu_sva *handle, 1062 const struct iommu_sva_ops *ops) 1063 { 1064 return -EINVAL; 1065 } 1066 1067 static inline int iommu_sva_get_pasid(struct iommu_sva *handle) 1068 { 1069 return IOMMU_PASID_INVALID; 1070 } 1071 1072 static inline int 1073 iommu_cache_invalidate(struct iommu_domain *domain, 1074 struct device *dev, 1075 struct iommu_cache_invalidate_info *inv_info) 1076 { 1077 return -ENODEV; 1078 } 1079 static inline int iommu_sva_bind_gpasid(struct iommu_domain *domain, 1080 struct device *dev, struct iommu_gpasid_bind_data *data) 1081 { 1082 return -ENODEV; 1083 } 1084 1085 static inline int iommu_sva_unbind_gpasid(struct iommu_domain *domain, 1086 struct device *dev, int pasid) 1087 { 1088 return -ENODEV; 1089 } 1090 1091 static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev) 1092 { 1093 return NULL; 1094 } 1095 #endif /* CONFIG_IOMMU_API */ 1096 1097 #ifdef CONFIG_IOMMU_DEBUGFS 1098 extern struct dentry *iommu_debugfs_dir; 1099 void iommu_debugfs_setup(void); 1100 #else 1101 static inline void iommu_debugfs_setup(void) {} 1102 #endif 1103 1104 #endif /* __LINUX_IOMMU_H */ 1105