1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc. 4 * Author: Joerg Roedel <[email protected]> 5 */ 6 7 #ifndef __LINUX_IOMMU_H 8 #define __LINUX_IOMMU_H 9 10 #include <linux/scatterlist.h> 11 #include <linux/device.h> 12 #include <linux/types.h> 13 #include <linux/errno.h> 14 #include <linux/err.h> 15 #include <linux/of.h> 16 #include <linux/ioasid.h> 17 #include <uapi/linux/iommu.h> 18 19 #define IOMMU_READ (1 << 0) 20 #define IOMMU_WRITE (1 << 1) 21 #define IOMMU_CACHE (1 << 2) /* DMA cache coherency */ 22 #define IOMMU_NOEXEC (1 << 3) 23 #define IOMMU_MMIO (1 << 4) /* e.g. things like MSI doorbells */ 24 /* 25 * Where the bus hardware includes a privilege level as part of its access type 26 * markings, and certain devices are capable of issuing transactions marked as 27 * either 'supervisor' or 'user', the IOMMU_PRIV flag requests that the other 28 * given permission flags only apply to accesses at the higher privilege level, 29 * and that unprivileged transactions should have as little access as possible. 30 * This would usually imply the same permissions as kernel mappings on the CPU, 31 * if the IOMMU page table format is equivalent. 32 */ 33 #define IOMMU_PRIV (1 << 5) 34 35 struct iommu_ops; 36 struct iommu_group; 37 struct bus_type; 38 struct device; 39 struct iommu_domain; 40 struct notifier_block; 41 struct iommu_sva; 42 struct iommu_fault_event; 43 struct iommu_dma_cookie; 44 45 /* iommu fault flags */ 46 #define IOMMU_FAULT_READ 0x0 47 #define IOMMU_FAULT_WRITE 0x1 48 49 typedef int (*iommu_fault_handler_t)(struct iommu_domain *, 50 struct device *, unsigned long, int, void *); 51 typedef int (*iommu_dev_fault_handler_t)(struct iommu_fault *, void *); 52 53 struct iommu_domain_geometry { 54 dma_addr_t aperture_start; /* First address that can be mapped */ 55 dma_addr_t aperture_end; /* Last address that can be mapped */ 56 bool force_aperture; /* DMA only allowed in mappable range? */ 57 }; 58 59 /* Domain feature flags */ 60 #define __IOMMU_DOMAIN_PAGING (1U << 0) /* Support for iommu_map/unmap */ 61 #define __IOMMU_DOMAIN_DMA_API (1U << 1) /* Domain for use in DMA-API 62 implementation */ 63 #define __IOMMU_DOMAIN_PT (1U << 2) /* Domain is identity mapped */ 64 #define __IOMMU_DOMAIN_DMA_FQ (1U << 3) /* DMA-API uses flush queue */ 65 66 /* 67 * This are the possible domain-types 68 * 69 * IOMMU_DOMAIN_BLOCKED - All DMA is blocked, can be used to isolate 70 * devices 71 * IOMMU_DOMAIN_IDENTITY - DMA addresses are system physical addresses 72 * IOMMU_DOMAIN_UNMANAGED - DMA mappings managed by IOMMU-API user, used 73 * for VMs 74 * IOMMU_DOMAIN_DMA - Internally used for DMA-API implementations. 75 * This flag allows IOMMU drivers to implement 76 * certain optimizations for these domains 77 * IOMMU_DOMAIN_DMA_FQ - As above, but definitely using batched TLB 78 * invalidation. 79 */ 80 #define IOMMU_DOMAIN_BLOCKED (0U) 81 #define IOMMU_DOMAIN_IDENTITY (__IOMMU_DOMAIN_PT) 82 #define IOMMU_DOMAIN_UNMANAGED (__IOMMU_DOMAIN_PAGING) 83 #define IOMMU_DOMAIN_DMA (__IOMMU_DOMAIN_PAGING | \ 84 __IOMMU_DOMAIN_DMA_API) 85 #define IOMMU_DOMAIN_DMA_FQ (__IOMMU_DOMAIN_PAGING | \ 86 __IOMMU_DOMAIN_DMA_API | \ 87 __IOMMU_DOMAIN_DMA_FQ) 88 89 struct iommu_domain { 90 unsigned type; 91 const struct iommu_ops *ops; 92 unsigned long pgsize_bitmap; /* Bitmap of page sizes in use */ 93 iommu_fault_handler_t handler; 94 void *handler_token; 95 struct iommu_domain_geometry geometry; 96 struct iommu_dma_cookie *iova_cookie; 97 }; 98 99 static inline bool iommu_is_dma_domain(struct iommu_domain *domain) 100 { 101 return domain->type & __IOMMU_DOMAIN_DMA_API; 102 } 103 104 enum iommu_cap { 105 IOMMU_CAP_CACHE_COHERENCY, /* IOMMU can enforce cache coherent DMA 106 transactions */ 107 IOMMU_CAP_INTR_REMAP, /* IOMMU supports interrupt isolation */ 108 IOMMU_CAP_NOEXEC, /* IOMMU_NOEXEC flag */ 109 }; 110 111 /* These are the possible reserved region types */ 112 enum iommu_resv_type { 113 /* Memory regions which must be mapped 1:1 at all times */ 114 IOMMU_RESV_DIRECT, 115 /* 116 * Memory regions which are advertised to be 1:1 but are 117 * commonly considered relaxable in some conditions, 118 * for instance in device assignment use case (USB, Graphics) 119 */ 120 IOMMU_RESV_DIRECT_RELAXABLE, 121 /* Arbitrary "never map this or give it to a device" address ranges */ 122 IOMMU_RESV_RESERVED, 123 /* Hardware MSI region (untranslated) */ 124 IOMMU_RESV_MSI, 125 /* Software-managed MSI translation window */ 126 IOMMU_RESV_SW_MSI, 127 }; 128 129 /** 130 * struct iommu_resv_region - descriptor for a reserved memory region 131 * @list: Linked list pointers 132 * @start: System physical start address of the region 133 * @length: Length of the region in bytes 134 * @prot: IOMMU Protection flags (READ/WRITE/...) 135 * @type: Type of the reserved region 136 */ 137 struct iommu_resv_region { 138 struct list_head list; 139 phys_addr_t start; 140 size_t length; 141 int prot; 142 enum iommu_resv_type type; 143 }; 144 145 /** 146 * enum iommu_dev_features - Per device IOMMU features 147 * @IOMMU_DEV_FEAT_AUX: Auxiliary domain feature 148 * @IOMMU_DEV_FEAT_SVA: Shared Virtual Addresses 149 * @IOMMU_DEV_FEAT_IOPF: I/O Page Faults such as PRI or Stall. Generally 150 * enabling %IOMMU_DEV_FEAT_SVA requires 151 * %IOMMU_DEV_FEAT_IOPF, but some devices manage I/O Page 152 * Faults themselves instead of relying on the IOMMU. When 153 * supported, this feature must be enabled before and 154 * disabled after %IOMMU_DEV_FEAT_SVA. 155 * 156 * Device drivers query whether a feature is supported using 157 * iommu_dev_has_feature(), and enable it using iommu_dev_enable_feature(). 158 */ 159 enum iommu_dev_features { 160 IOMMU_DEV_FEAT_AUX, 161 IOMMU_DEV_FEAT_SVA, 162 IOMMU_DEV_FEAT_IOPF, 163 }; 164 165 #define IOMMU_PASID_INVALID (-1U) 166 167 #ifdef CONFIG_IOMMU_API 168 169 /** 170 * struct iommu_iotlb_gather - Range information for a pending IOTLB flush 171 * 172 * @start: IOVA representing the start of the range to be flushed 173 * @end: IOVA representing the end of the range to be flushed (inclusive) 174 * @pgsize: The interval at which to perform the flush 175 * @freelist: Removed pages to free after sync 176 * @queued: Indicates that the flush will be queued 177 * 178 * This structure is intended to be updated by multiple calls to the 179 * ->unmap() function in struct iommu_ops before eventually being passed 180 * into ->iotlb_sync(). Drivers can add pages to @freelist to be freed after 181 * ->iotlb_sync() or ->iotlb_flush_all() have cleared all cached references to 182 * them. @queued is set to indicate when ->iotlb_flush_all() will be called 183 * later instead of ->iotlb_sync(), so drivers may optimise accordingly. 184 */ 185 struct iommu_iotlb_gather { 186 unsigned long start; 187 unsigned long end; 188 size_t pgsize; 189 struct list_head freelist; 190 bool queued; 191 }; 192 193 /** 194 * struct iommu_ops - iommu ops and capabilities 195 * @capable: check capability 196 * @domain_alloc: allocate iommu domain 197 * @domain_free: free iommu domain 198 * @attach_dev: attach device to an iommu domain 199 * @detach_dev: detach device from an iommu domain 200 * @map: map a physically contiguous memory region to an iommu domain 201 * @map_pages: map a physically contiguous set of pages of the same size to 202 * an iommu domain. 203 * @unmap: unmap a physically contiguous memory region from an iommu domain 204 * @unmap_pages: unmap a number of pages of the same size from an iommu domain 205 * @flush_iotlb_all: Synchronously flush all hardware TLBs for this domain 206 * @iotlb_sync_map: Sync mappings created recently using @map to the hardware 207 * @iotlb_sync: Flush all queued ranges from the hardware TLBs and empty flush 208 * queue 209 * @iova_to_phys: translate iova to physical address 210 * @probe_device: Add device to iommu driver handling 211 * @release_device: Remove device from iommu driver handling 212 * @probe_finalize: Do final setup work after the device is added to an IOMMU 213 * group and attached to the groups domain 214 * @device_group: find iommu group for a particular device 215 * @enable_nesting: Enable nesting 216 * @set_pgtable_quirks: Set io page table quirks (IO_PGTABLE_QUIRK_*) 217 * @get_resv_regions: Request list of reserved regions for a device 218 * @put_resv_regions: Free list of reserved regions for a device 219 * @apply_resv_region: Temporary helper call-back for iova reserved ranges 220 * @of_xlate: add OF master IDs to iommu grouping 221 * @is_attach_deferred: Check if domain attach should be deferred from iommu 222 * driver init to device driver init (default no) 223 * @dev_has/enable/disable_feat: per device entries to check/enable/disable 224 * iommu specific features. 225 * @dev_feat_enabled: check enabled feature 226 * @aux_attach/detach_dev: aux-domain specific attach/detach entries. 227 * @aux_get_pasid: get the pasid given an aux-domain 228 * @sva_bind: Bind process address space to device 229 * @sva_unbind: Unbind process address space from device 230 * @sva_get_pasid: Get PASID associated to a SVA handle 231 * @page_response: handle page request response 232 * @cache_invalidate: invalidate translation caches 233 * @sva_bind_gpasid: bind guest pasid and mm 234 * @sva_unbind_gpasid: unbind guest pasid and mm 235 * @def_domain_type: device default domain type, return value: 236 * - IOMMU_DOMAIN_IDENTITY: must use an identity domain 237 * - IOMMU_DOMAIN_DMA: must use a dma domain 238 * - 0: use the default setting 239 * @pgsize_bitmap: bitmap of all possible supported page sizes 240 * @owner: Driver module providing these ops 241 */ 242 struct iommu_ops { 243 bool (*capable)(enum iommu_cap); 244 245 /* Domain allocation and freeing by the iommu driver */ 246 struct iommu_domain *(*domain_alloc)(unsigned iommu_domain_type); 247 void (*domain_free)(struct iommu_domain *); 248 249 int (*attach_dev)(struct iommu_domain *domain, struct device *dev); 250 void (*detach_dev)(struct iommu_domain *domain, struct device *dev); 251 int (*map)(struct iommu_domain *domain, unsigned long iova, 252 phys_addr_t paddr, size_t size, int prot, gfp_t gfp); 253 int (*map_pages)(struct iommu_domain *domain, unsigned long iova, 254 phys_addr_t paddr, size_t pgsize, size_t pgcount, 255 int prot, gfp_t gfp, size_t *mapped); 256 size_t (*unmap)(struct iommu_domain *domain, unsigned long iova, 257 size_t size, struct iommu_iotlb_gather *iotlb_gather); 258 size_t (*unmap_pages)(struct iommu_domain *domain, unsigned long iova, 259 size_t pgsize, size_t pgcount, 260 struct iommu_iotlb_gather *iotlb_gather); 261 void (*flush_iotlb_all)(struct iommu_domain *domain); 262 void (*iotlb_sync_map)(struct iommu_domain *domain, unsigned long iova, 263 size_t size); 264 void (*iotlb_sync)(struct iommu_domain *domain, 265 struct iommu_iotlb_gather *iotlb_gather); 266 phys_addr_t (*iova_to_phys)(struct iommu_domain *domain, dma_addr_t iova); 267 struct iommu_device *(*probe_device)(struct device *dev); 268 void (*release_device)(struct device *dev); 269 void (*probe_finalize)(struct device *dev); 270 struct iommu_group *(*device_group)(struct device *dev); 271 int (*enable_nesting)(struct iommu_domain *domain); 272 int (*set_pgtable_quirks)(struct iommu_domain *domain, 273 unsigned long quirks); 274 275 /* Request/Free a list of reserved regions for a device */ 276 void (*get_resv_regions)(struct device *dev, struct list_head *list); 277 void (*put_resv_regions)(struct device *dev, struct list_head *list); 278 void (*apply_resv_region)(struct device *dev, 279 struct iommu_domain *domain, 280 struct iommu_resv_region *region); 281 282 int (*of_xlate)(struct device *dev, struct of_phandle_args *args); 283 bool (*is_attach_deferred)(struct iommu_domain *domain, struct device *dev); 284 285 /* Per device IOMMU features */ 286 bool (*dev_has_feat)(struct device *dev, enum iommu_dev_features f); 287 bool (*dev_feat_enabled)(struct device *dev, enum iommu_dev_features f); 288 int (*dev_enable_feat)(struct device *dev, enum iommu_dev_features f); 289 int (*dev_disable_feat)(struct device *dev, enum iommu_dev_features f); 290 291 /* Aux-domain specific attach/detach entries */ 292 int (*aux_attach_dev)(struct iommu_domain *domain, struct device *dev); 293 void (*aux_detach_dev)(struct iommu_domain *domain, struct device *dev); 294 int (*aux_get_pasid)(struct iommu_domain *domain, struct device *dev); 295 296 struct iommu_sva *(*sva_bind)(struct device *dev, struct mm_struct *mm, 297 void *drvdata); 298 void (*sva_unbind)(struct iommu_sva *handle); 299 u32 (*sva_get_pasid)(struct iommu_sva *handle); 300 301 int (*page_response)(struct device *dev, 302 struct iommu_fault_event *evt, 303 struct iommu_page_response *msg); 304 int (*cache_invalidate)(struct iommu_domain *domain, struct device *dev, 305 struct iommu_cache_invalidate_info *inv_info); 306 int (*sva_bind_gpasid)(struct iommu_domain *domain, 307 struct device *dev, struct iommu_gpasid_bind_data *data); 308 309 int (*sva_unbind_gpasid)(struct device *dev, u32 pasid); 310 311 int (*def_domain_type)(struct device *dev); 312 313 unsigned long pgsize_bitmap; 314 struct module *owner; 315 }; 316 317 /** 318 * struct iommu_device - IOMMU core representation of one IOMMU hardware 319 * instance 320 * @list: Used by the iommu-core to keep a list of registered iommus 321 * @ops: iommu-ops for talking to this iommu 322 * @dev: struct device for sysfs handling 323 */ 324 struct iommu_device { 325 struct list_head list; 326 const struct iommu_ops *ops; 327 struct fwnode_handle *fwnode; 328 struct device *dev; 329 }; 330 331 /** 332 * struct iommu_fault_event - Generic fault event 333 * 334 * Can represent recoverable faults such as a page requests or 335 * unrecoverable faults such as DMA or IRQ remapping faults. 336 * 337 * @fault: fault descriptor 338 * @list: pending fault event list, used for tracking responses 339 */ 340 struct iommu_fault_event { 341 struct iommu_fault fault; 342 struct list_head list; 343 }; 344 345 /** 346 * struct iommu_fault_param - per-device IOMMU fault data 347 * @handler: Callback function to handle IOMMU faults at device level 348 * @data: handler private data 349 * @faults: holds the pending faults which needs response 350 * @lock: protect pending faults list 351 */ 352 struct iommu_fault_param { 353 iommu_dev_fault_handler_t handler; 354 void *data; 355 struct list_head faults; 356 struct mutex lock; 357 }; 358 359 /** 360 * struct dev_iommu - Collection of per-device IOMMU data 361 * 362 * @fault_param: IOMMU detected device fault reporting data 363 * @iopf_param: I/O Page Fault queue and data 364 * @fwspec: IOMMU fwspec data 365 * @iommu_dev: IOMMU device this device is linked to 366 * @priv: IOMMU Driver private data 367 * 368 * TODO: migrate other per device data pointers under iommu_dev_data, e.g. 369 * struct iommu_group *iommu_group; 370 */ 371 struct dev_iommu { 372 struct mutex lock; 373 struct iommu_fault_param *fault_param; 374 struct iopf_device_param *iopf_param; 375 struct iommu_fwspec *fwspec; 376 struct iommu_device *iommu_dev; 377 void *priv; 378 }; 379 380 int iommu_device_register(struct iommu_device *iommu, 381 const struct iommu_ops *ops, 382 struct device *hwdev); 383 void iommu_device_unregister(struct iommu_device *iommu); 384 int iommu_device_sysfs_add(struct iommu_device *iommu, 385 struct device *parent, 386 const struct attribute_group **groups, 387 const char *fmt, ...) __printf(4, 5); 388 void iommu_device_sysfs_remove(struct iommu_device *iommu); 389 int iommu_device_link(struct iommu_device *iommu, struct device *link); 390 void iommu_device_unlink(struct iommu_device *iommu, struct device *link); 391 int iommu_deferred_attach(struct device *dev, struct iommu_domain *domain); 392 393 static inline struct iommu_device *dev_to_iommu_device(struct device *dev) 394 { 395 return (struct iommu_device *)dev_get_drvdata(dev); 396 } 397 398 static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather) 399 { 400 *gather = (struct iommu_iotlb_gather) { 401 .start = ULONG_MAX, 402 .freelist = LIST_HEAD_INIT(gather->freelist), 403 }; 404 } 405 406 #define IOMMU_GROUP_NOTIFY_ADD_DEVICE 1 /* Device added */ 407 #define IOMMU_GROUP_NOTIFY_DEL_DEVICE 2 /* Pre Device removed */ 408 #define IOMMU_GROUP_NOTIFY_BIND_DRIVER 3 /* Pre Driver bind */ 409 #define IOMMU_GROUP_NOTIFY_BOUND_DRIVER 4 /* Post Driver bind */ 410 #define IOMMU_GROUP_NOTIFY_UNBIND_DRIVER 5 /* Pre Driver unbind */ 411 #define IOMMU_GROUP_NOTIFY_UNBOUND_DRIVER 6 /* Post Driver unbind */ 412 413 extern int bus_set_iommu(struct bus_type *bus, const struct iommu_ops *ops); 414 extern int bus_iommu_probe(struct bus_type *bus); 415 extern bool iommu_present(struct bus_type *bus); 416 extern bool iommu_capable(struct bus_type *bus, enum iommu_cap cap); 417 extern struct iommu_domain *iommu_domain_alloc(struct bus_type *bus); 418 extern struct iommu_group *iommu_group_get_by_id(int id); 419 extern void iommu_domain_free(struct iommu_domain *domain); 420 extern int iommu_attach_device(struct iommu_domain *domain, 421 struct device *dev); 422 extern void iommu_detach_device(struct iommu_domain *domain, 423 struct device *dev); 424 extern int iommu_uapi_cache_invalidate(struct iommu_domain *domain, 425 struct device *dev, 426 void __user *uinfo); 427 428 extern int iommu_uapi_sva_bind_gpasid(struct iommu_domain *domain, 429 struct device *dev, void __user *udata); 430 extern int iommu_uapi_sva_unbind_gpasid(struct iommu_domain *domain, 431 struct device *dev, void __user *udata); 432 extern int iommu_sva_unbind_gpasid(struct iommu_domain *domain, 433 struct device *dev, ioasid_t pasid); 434 extern struct iommu_domain *iommu_get_domain_for_dev(struct device *dev); 435 extern struct iommu_domain *iommu_get_dma_domain(struct device *dev); 436 extern int iommu_map(struct iommu_domain *domain, unsigned long iova, 437 phys_addr_t paddr, size_t size, int prot); 438 extern int iommu_map_atomic(struct iommu_domain *domain, unsigned long iova, 439 phys_addr_t paddr, size_t size, int prot); 440 extern size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, 441 size_t size); 442 extern size_t iommu_unmap_fast(struct iommu_domain *domain, 443 unsigned long iova, size_t size, 444 struct iommu_iotlb_gather *iotlb_gather); 445 extern ssize_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova, 446 struct scatterlist *sg, unsigned int nents, int prot); 447 extern ssize_t iommu_map_sg_atomic(struct iommu_domain *domain, 448 unsigned long iova, struct scatterlist *sg, 449 unsigned int nents, int prot); 450 extern phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova); 451 extern void iommu_set_fault_handler(struct iommu_domain *domain, 452 iommu_fault_handler_t handler, void *token); 453 454 extern void iommu_get_resv_regions(struct device *dev, struct list_head *list); 455 extern void iommu_put_resv_regions(struct device *dev, struct list_head *list); 456 extern void generic_iommu_put_resv_regions(struct device *dev, 457 struct list_head *list); 458 extern void iommu_set_default_passthrough(bool cmd_line); 459 extern void iommu_set_default_translated(bool cmd_line); 460 extern bool iommu_default_passthrough(void); 461 extern struct iommu_resv_region * 462 iommu_alloc_resv_region(phys_addr_t start, size_t length, int prot, 463 enum iommu_resv_type type); 464 extern int iommu_get_group_resv_regions(struct iommu_group *group, 465 struct list_head *head); 466 467 extern int iommu_attach_group(struct iommu_domain *domain, 468 struct iommu_group *group); 469 extern void iommu_detach_group(struct iommu_domain *domain, 470 struct iommu_group *group); 471 extern struct iommu_group *iommu_group_alloc(void); 472 extern void *iommu_group_get_iommudata(struct iommu_group *group); 473 extern void iommu_group_set_iommudata(struct iommu_group *group, 474 void *iommu_data, 475 void (*release)(void *iommu_data)); 476 extern int iommu_group_set_name(struct iommu_group *group, const char *name); 477 extern int iommu_group_add_device(struct iommu_group *group, 478 struct device *dev); 479 extern void iommu_group_remove_device(struct device *dev); 480 extern int iommu_group_for_each_dev(struct iommu_group *group, void *data, 481 int (*fn)(struct device *, void *)); 482 extern struct iommu_group *iommu_group_get(struct device *dev); 483 extern struct iommu_group *iommu_group_ref_get(struct iommu_group *group); 484 extern void iommu_group_put(struct iommu_group *group); 485 extern int iommu_group_register_notifier(struct iommu_group *group, 486 struct notifier_block *nb); 487 extern int iommu_group_unregister_notifier(struct iommu_group *group, 488 struct notifier_block *nb); 489 extern int iommu_register_device_fault_handler(struct device *dev, 490 iommu_dev_fault_handler_t handler, 491 void *data); 492 493 extern int iommu_unregister_device_fault_handler(struct device *dev); 494 495 extern int iommu_report_device_fault(struct device *dev, 496 struct iommu_fault_event *evt); 497 extern int iommu_page_response(struct device *dev, 498 struct iommu_page_response *msg); 499 500 extern int iommu_group_id(struct iommu_group *group); 501 extern struct iommu_domain *iommu_group_default_domain(struct iommu_group *); 502 503 int iommu_enable_nesting(struct iommu_domain *domain); 504 int iommu_set_pgtable_quirks(struct iommu_domain *domain, 505 unsigned long quirks); 506 507 void iommu_set_dma_strict(void); 508 509 extern int report_iommu_fault(struct iommu_domain *domain, struct device *dev, 510 unsigned long iova, int flags); 511 512 static inline void iommu_flush_iotlb_all(struct iommu_domain *domain) 513 { 514 if (domain->ops->flush_iotlb_all) 515 domain->ops->flush_iotlb_all(domain); 516 } 517 518 static inline void iommu_iotlb_sync(struct iommu_domain *domain, 519 struct iommu_iotlb_gather *iotlb_gather) 520 { 521 if (domain->ops->iotlb_sync) 522 domain->ops->iotlb_sync(domain, iotlb_gather); 523 524 iommu_iotlb_gather_init(iotlb_gather); 525 } 526 527 /** 528 * iommu_iotlb_gather_is_disjoint - Checks whether a new range is disjoint 529 * 530 * @gather: TLB gather data 531 * @iova: start of page to invalidate 532 * @size: size of page to invalidate 533 * 534 * Helper for IOMMU drivers to check whether a new range and the gathered range 535 * are disjoint. For many IOMMUs, flushing the IOMMU in this case is better 536 * than merging the two, which might lead to unnecessary invalidations. 537 */ 538 static inline 539 bool iommu_iotlb_gather_is_disjoint(struct iommu_iotlb_gather *gather, 540 unsigned long iova, size_t size) 541 { 542 unsigned long start = iova, end = start + size - 1; 543 544 return gather->end != 0 && 545 (end + 1 < gather->start || start > gather->end + 1); 546 } 547 548 549 /** 550 * iommu_iotlb_gather_add_range - Gather for address-based TLB invalidation 551 * @gather: TLB gather data 552 * @iova: start of page to invalidate 553 * @size: size of page to invalidate 554 * 555 * Helper for IOMMU drivers to build arbitrarily-sized invalidation commands 556 * where only the address range matters, and simply minimising intermediate 557 * syncs is preferred. 558 */ 559 static inline void iommu_iotlb_gather_add_range(struct iommu_iotlb_gather *gather, 560 unsigned long iova, size_t size) 561 { 562 unsigned long end = iova + size - 1; 563 564 if (gather->start > iova) 565 gather->start = iova; 566 if (gather->end < end) 567 gather->end = end; 568 } 569 570 /** 571 * iommu_iotlb_gather_add_page - Gather for page-based TLB invalidation 572 * @domain: IOMMU domain to be invalidated 573 * @gather: TLB gather data 574 * @iova: start of page to invalidate 575 * @size: size of page to invalidate 576 * 577 * Helper for IOMMU drivers to build invalidation commands based on individual 578 * pages, or with page size/table level hints which cannot be gathered if they 579 * differ. 580 */ 581 static inline void iommu_iotlb_gather_add_page(struct iommu_domain *domain, 582 struct iommu_iotlb_gather *gather, 583 unsigned long iova, size_t size) 584 { 585 /* 586 * If the new page is disjoint from the current range or is mapped at 587 * a different granularity, then sync the TLB so that the gather 588 * structure can be rewritten. 589 */ 590 if ((gather->pgsize && gather->pgsize != size) || 591 iommu_iotlb_gather_is_disjoint(gather, iova, size)) 592 iommu_iotlb_sync(domain, gather); 593 594 gather->pgsize = size; 595 iommu_iotlb_gather_add_range(gather, iova, size); 596 } 597 598 static inline bool iommu_iotlb_gather_queued(struct iommu_iotlb_gather *gather) 599 { 600 return gather && gather->queued; 601 } 602 603 /* PCI device grouping function */ 604 extern struct iommu_group *pci_device_group(struct device *dev); 605 /* Generic device grouping function */ 606 extern struct iommu_group *generic_device_group(struct device *dev); 607 /* FSL-MC device grouping function */ 608 struct iommu_group *fsl_mc_device_group(struct device *dev); 609 610 /** 611 * struct iommu_fwspec - per-device IOMMU instance data 612 * @ops: ops for this device's IOMMU 613 * @iommu_fwnode: firmware handle for this device's IOMMU 614 * @flags: IOMMU_FWSPEC_* flags 615 * @num_ids: number of associated device IDs 616 * @ids: IDs which this device may present to the IOMMU 617 */ 618 struct iommu_fwspec { 619 const struct iommu_ops *ops; 620 struct fwnode_handle *iommu_fwnode; 621 u32 flags; 622 unsigned int num_ids; 623 u32 ids[]; 624 }; 625 626 /* ATS is supported */ 627 #define IOMMU_FWSPEC_PCI_RC_ATS (1 << 0) 628 629 /** 630 * struct iommu_sva - handle to a device-mm bond 631 */ 632 struct iommu_sva { 633 struct device *dev; 634 }; 635 636 int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode, 637 const struct iommu_ops *ops); 638 void iommu_fwspec_free(struct device *dev); 639 int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids); 640 const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode); 641 642 static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev) 643 { 644 if (dev->iommu) 645 return dev->iommu->fwspec; 646 else 647 return NULL; 648 } 649 650 static inline void dev_iommu_fwspec_set(struct device *dev, 651 struct iommu_fwspec *fwspec) 652 { 653 dev->iommu->fwspec = fwspec; 654 } 655 656 static inline void *dev_iommu_priv_get(struct device *dev) 657 { 658 if (dev->iommu) 659 return dev->iommu->priv; 660 else 661 return NULL; 662 } 663 664 static inline void dev_iommu_priv_set(struct device *dev, void *priv) 665 { 666 dev->iommu->priv = priv; 667 } 668 669 int iommu_probe_device(struct device *dev); 670 void iommu_release_device(struct device *dev); 671 672 int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features f); 673 int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features f); 674 bool iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features f); 675 int iommu_aux_attach_device(struct iommu_domain *domain, struct device *dev); 676 void iommu_aux_detach_device(struct iommu_domain *domain, struct device *dev); 677 int iommu_aux_get_pasid(struct iommu_domain *domain, struct device *dev); 678 679 struct iommu_sva *iommu_sva_bind_device(struct device *dev, 680 struct mm_struct *mm, 681 void *drvdata); 682 void iommu_sva_unbind_device(struct iommu_sva *handle); 683 u32 iommu_sva_get_pasid(struct iommu_sva *handle); 684 685 #else /* CONFIG_IOMMU_API */ 686 687 struct iommu_ops {}; 688 struct iommu_group {}; 689 struct iommu_fwspec {}; 690 struct iommu_device {}; 691 struct iommu_fault_param {}; 692 struct iommu_iotlb_gather {}; 693 694 static inline bool iommu_present(struct bus_type *bus) 695 { 696 return false; 697 } 698 699 static inline bool iommu_capable(struct bus_type *bus, enum iommu_cap cap) 700 { 701 return false; 702 } 703 704 static inline struct iommu_domain *iommu_domain_alloc(struct bus_type *bus) 705 { 706 return NULL; 707 } 708 709 static inline struct iommu_group *iommu_group_get_by_id(int id) 710 { 711 return NULL; 712 } 713 714 static inline void iommu_domain_free(struct iommu_domain *domain) 715 { 716 } 717 718 static inline int iommu_attach_device(struct iommu_domain *domain, 719 struct device *dev) 720 { 721 return -ENODEV; 722 } 723 724 static inline void iommu_detach_device(struct iommu_domain *domain, 725 struct device *dev) 726 { 727 } 728 729 static inline struct iommu_domain *iommu_get_domain_for_dev(struct device *dev) 730 { 731 return NULL; 732 } 733 734 static inline int iommu_map(struct iommu_domain *domain, unsigned long iova, 735 phys_addr_t paddr, size_t size, int prot) 736 { 737 return -ENODEV; 738 } 739 740 static inline int iommu_map_atomic(struct iommu_domain *domain, 741 unsigned long iova, phys_addr_t paddr, 742 size_t size, int prot) 743 { 744 return -ENODEV; 745 } 746 747 static inline size_t iommu_unmap(struct iommu_domain *domain, 748 unsigned long iova, size_t size) 749 { 750 return 0; 751 } 752 753 static inline size_t iommu_unmap_fast(struct iommu_domain *domain, 754 unsigned long iova, int gfp_order, 755 struct iommu_iotlb_gather *iotlb_gather) 756 { 757 return 0; 758 } 759 760 static inline ssize_t iommu_map_sg(struct iommu_domain *domain, 761 unsigned long iova, struct scatterlist *sg, 762 unsigned int nents, int prot) 763 { 764 return -ENODEV; 765 } 766 767 static inline ssize_t iommu_map_sg_atomic(struct iommu_domain *domain, 768 unsigned long iova, struct scatterlist *sg, 769 unsigned int nents, int prot) 770 { 771 return -ENODEV; 772 } 773 774 static inline void iommu_flush_iotlb_all(struct iommu_domain *domain) 775 { 776 } 777 778 static inline void iommu_iotlb_sync(struct iommu_domain *domain, 779 struct iommu_iotlb_gather *iotlb_gather) 780 { 781 } 782 783 static inline phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) 784 { 785 return 0; 786 } 787 788 static inline void iommu_set_fault_handler(struct iommu_domain *domain, 789 iommu_fault_handler_t handler, void *token) 790 { 791 } 792 793 static inline void iommu_get_resv_regions(struct device *dev, 794 struct list_head *list) 795 { 796 } 797 798 static inline void iommu_put_resv_regions(struct device *dev, 799 struct list_head *list) 800 { 801 } 802 803 static inline int iommu_get_group_resv_regions(struct iommu_group *group, 804 struct list_head *head) 805 { 806 return -ENODEV; 807 } 808 809 static inline void iommu_set_default_passthrough(bool cmd_line) 810 { 811 } 812 813 static inline void iommu_set_default_translated(bool cmd_line) 814 { 815 } 816 817 static inline bool iommu_default_passthrough(void) 818 { 819 return true; 820 } 821 822 static inline int iommu_attach_group(struct iommu_domain *domain, 823 struct iommu_group *group) 824 { 825 return -ENODEV; 826 } 827 828 static inline void iommu_detach_group(struct iommu_domain *domain, 829 struct iommu_group *group) 830 { 831 } 832 833 static inline struct iommu_group *iommu_group_alloc(void) 834 { 835 return ERR_PTR(-ENODEV); 836 } 837 838 static inline void *iommu_group_get_iommudata(struct iommu_group *group) 839 { 840 return NULL; 841 } 842 843 static inline void iommu_group_set_iommudata(struct iommu_group *group, 844 void *iommu_data, 845 void (*release)(void *iommu_data)) 846 { 847 } 848 849 static inline int iommu_group_set_name(struct iommu_group *group, 850 const char *name) 851 { 852 return -ENODEV; 853 } 854 855 static inline int iommu_group_add_device(struct iommu_group *group, 856 struct device *dev) 857 { 858 return -ENODEV; 859 } 860 861 static inline void iommu_group_remove_device(struct device *dev) 862 { 863 } 864 865 static inline int iommu_group_for_each_dev(struct iommu_group *group, 866 void *data, 867 int (*fn)(struct device *, void *)) 868 { 869 return -ENODEV; 870 } 871 872 static inline struct iommu_group *iommu_group_get(struct device *dev) 873 { 874 return NULL; 875 } 876 877 static inline void iommu_group_put(struct iommu_group *group) 878 { 879 } 880 881 static inline int iommu_group_register_notifier(struct iommu_group *group, 882 struct notifier_block *nb) 883 { 884 return -ENODEV; 885 } 886 887 static inline int iommu_group_unregister_notifier(struct iommu_group *group, 888 struct notifier_block *nb) 889 { 890 return 0; 891 } 892 893 static inline 894 int iommu_register_device_fault_handler(struct device *dev, 895 iommu_dev_fault_handler_t handler, 896 void *data) 897 { 898 return -ENODEV; 899 } 900 901 static inline int iommu_unregister_device_fault_handler(struct device *dev) 902 { 903 return 0; 904 } 905 906 static inline 907 int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt) 908 { 909 return -ENODEV; 910 } 911 912 static inline int iommu_page_response(struct device *dev, 913 struct iommu_page_response *msg) 914 { 915 return -ENODEV; 916 } 917 918 static inline int iommu_group_id(struct iommu_group *group) 919 { 920 return -ENODEV; 921 } 922 923 static inline int iommu_set_pgtable_quirks(struct iommu_domain *domain, 924 unsigned long quirks) 925 { 926 return 0; 927 } 928 929 static inline int iommu_device_register(struct iommu_device *iommu, 930 const struct iommu_ops *ops, 931 struct device *hwdev) 932 { 933 return -ENODEV; 934 } 935 936 static inline struct iommu_device *dev_to_iommu_device(struct device *dev) 937 { 938 return NULL; 939 } 940 941 static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather) 942 { 943 } 944 945 static inline void iommu_iotlb_gather_add_page(struct iommu_domain *domain, 946 struct iommu_iotlb_gather *gather, 947 unsigned long iova, size_t size) 948 { 949 } 950 951 static inline bool iommu_iotlb_gather_queued(struct iommu_iotlb_gather *gather) 952 { 953 return false; 954 } 955 956 static inline void iommu_device_unregister(struct iommu_device *iommu) 957 { 958 } 959 960 static inline int iommu_device_sysfs_add(struct iommu_device *iommu, 961 struct device *parent, 962 const struct attribute_group **groups, 963 const char *fmt, ...) 964 { 965 return -ENODEV; 966 } 967 968 static inline void iommu_device_sysfs_remove(struct iommu_device *iommu) 969 { 970 } 971 972 static inline int iommu_device_link(struct device *dev, struct device *link) 973 { 974 return -EINVAL; 975 } 976 977 static inline void iommu_device_unlink(struct device *dev, struct device *link) 978 { 979 } 980 981 static inline int iommu_fwspec_init(struct device *dev, 982 struct fwnode_handle *iommu_fwnode, 983 const struct iommu_ops *ops) 984 { 985 return -ENODEV; 986 } 987 988 static inline void iommu_fwspec_free(struct device *dev) 989 { 990 } 991 992 static inline int iommu_fwspec_add_ids(struct device *dev, u32 *ids, 993 int num_ids) 994 { 995 return -ENODEV; 996 } 997 998 static inline 999 const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode) 1000 { 1001 return NULL; 1002 } 1003 1004 static inline bool 1005 iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features feat) 1006 { 1007 return false; 1008 } 1009 1010 static inline int 1011 iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features feat) 1012 { 1013 return -ENODEV; 1014 } 1015 1016 static inline int 1017 iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat) 1018 { 1019 return -ENODEV; 1020 } 1021 1022 static inline int 1023 iommu_aux_attach_device(struct iommu_domain *domain, struct device *dev) 1024 { 1025 return -ENODEV; 1026 } 1027 1028 static inline void 1029 iommu_aux_detach_device(struct iommu_domain *domain, struct device *dev) 1030 { 1031 } 1032 1033 static inline int 1034 iommu_aux_get_pasid(struct iommu_domain *domain, struct device *dev) 1035 { 1036 return -ENODEV; 1037 } 1038 1039 static inline struct iommu_sva * 1040 iommu_sva_bind_device(struct device *dev, struct mm_struct *mm, void *drvdata) 1041 { 1042 return NULL; 1043 } 1044 1045 static inline void iommu_sva_unbind_device(struct iommu_sva *handle) 1046 { 1047 } 1048 1049 static inline u32 iommu_sva_get_pasid(struct iommu_sva *handle) 1050 { 1051 return IOMMU_PASID_INVALID; 1052 } 1053 1054 static inline int 1055 iommu_uapi_cache_invalidate(struct iommu_domain *domain, 1056 struct device *dev, 1057 struct iommu_cache_invalidate_info *inv_info) 1058 { 1059 return -ENODEV; 1060 } 1061 1062 static inline int iommu_uapi_sva_bind_gpasid(struct iommu_domain *domain, 1063 struct device *dev, void __user *udata) 1064 { 1065 return -ENODEV; 1066 } 1067 1068 static inline int iommu_uapi_sva_unbind_gpasid(struct iommu_domain *domain, 1069 struct device *dev, void __user *udata) 1070 { 1071 return -ENODEV; 1072 } 1073 1074 static inline int iommu_sva_unbind_gpasid(struct iommu_domain *domain, 1075 struct device *dev, 1076 ioasid_t pasid) 1077 { 1078 return -ENODEV; 1079 } 1080 1081 static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev) 1082 { 1083 return NULL; 1084 } 1085 #endif /* CONFIG_IOMMU_API */ 1086 1087 /** 1088 * iommu_map_sgtable - Map the given buffer to the IOMMU domain 1089 * @domain: The IOMMU domain to perform the mapping 1090 * @iova: The start address to map the buffer 1091 * @sgt: The sg_table object describing the buffer 1092 * @prot: IOMMU protection bits 1093 * 1094 * Creates a mapping at @iova for the buffer described by a scatterlist 1095 * stored in the given sg_table object in the provided IOMMU domain. 1096 */ 1097 static inline size_t iommu_map_sgtable(struct iommu_domain *domain, 1098 unsigned long iova, struct sg_table *sgt, int prot) 1099 { 1100 return iommu_map_sg(domain, iova, sgt->sgl, sgt->orig_nents, prot); 1101 } 1102 1103 #ifdef CONFIG_IOMMU_DEBUGFS 1104 extern struct dentry *iommu_debugfs_dir; 1105 void iommu_debugfs_setup(void); 1106 #else 1107 static inline void iommu_debugfs_setup(void) {} 1108 #endif 1109 1110 #endif /* __LINUX_IOMMU_H */ 1111