1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc. 4 * Author: Joerg Roedel <[email protected]> 5 */ 6 7 #ifndef __LINUX_IOMMU_H 8 #define __LINUX_IOMMU_H 9 10 #include <linux/scatterlist.h> 11 #include <linux/device.h> 12 #include <linux/types.h> 13 #include <linux/errno.h> 14 #include <linux/err.h> 15 #include <linux/of.h> 16 #include <linux/ioasid.h> 17 #include <uapi/linux/iommu.h> 18 19 #define IOMMU_READ (1 << 0) 20 #define IOMMU_WRITE (1 << 1) 21 #define IOMMU_CACHE (1 << 2) /* DMA cache coherency */ 22 #define IOMMU_NOEXEC (1 << 3) 23 #define IOMMU_MMIO (1 << 4) /* e.g. things like MSI doorbells */ 24 /* 25 * Where the bus hardware includes a privilege level as part of its access type 26 * markings, and certain devices are capable of issuing transactions marked as 27 * either 'supervisor' or 'user', the IOMMU_PRIV flag requests that the other 28 * given permission flags only apply to accesses at the higher privilege level, 29 * and that unprivileged transactions should have as little access as possible. 30 * This would usually imply the same permissions as kernel mappings on the CPU, 31 * if the IOMMU page table format is equivalent. 32 */ 33 #define IOMMU_PRIV (1 << 5) 34 35 struct iommu_ops; 36 struct iommu_group; 37 struct bus_type; 38 struct device; 39 struct iommu_domain; 40 struct notifier_block; 41 struct iommu_sva; 42 struct iommu_fault_event; 43 44 /* iommu fault flags */ 45 #define IOMMU_FAULT_READ 0x0 46 #define IOMMU_FAULT_WRITE 0x1 47 48 typedef int (*iommu_fault_handler_t)(struct iommu_domain *, 49 struct device *, unsigned long, int, void *); 50 typedef int (*iommu_dev_fault_handler_t)(struct iommu_fault *, void *); 51 52 struct iommu_domain_geometry { 53 dma_addr_t aperture_start; /* First address that can be mapped */ 54 dma_addr_t aperture_end; /* Last address that can be mapped */ 55 bool force_aperture; /* DMA only allowed in mappable range? */ 56 }; 57 58 /* Domain feature flags */ 59 #define __IOMMU_DOMAIN_PAGING (1U << 0) /* Support for iommu_map/unmap */ 60 #define __IOMMU_DOMAIN_DMA_API (1U << 1) /* Domain for use in DMA-API 61 implementation */ 62 #define __IOMMU_DOMAIN_PT (1U << 2) /* Domain is identity mapped */ 63 64 /* 65 * This are the possible domain-types 66 * 67 * IOMMU_DOMAIN_BLOCKED - All DMA is blocked, can be used to isolate 68 * devices 69 * IOMMU_DOMAIN_IDENTITY - DMA addresses are system physical addresses 70 * IOMMU_DOMAIN_UNMANAGED - DMA mappings managed by IOMMU-API user, used 71 * for VMs 72 * IOMMU_DOMAIN_DMA - Internally used for DMA-API implementations. 73 * This flag allows IOMMU drivers to implement 74 * certain optimizations for these domains 75 */ 76 #define IOMMU_DOMAIN_BLOCKED (0U) 77 #define IOMMU_DOMAIN_IDENTITY (__IOMMU_DOMAIN_PT) 78 #define IOMMU_DOMAIN_UNMANAGED (__IOMMU_DOMAIN_PAGING) 79 #define IOMMU_DOMAIN_DMA (__IOMMU_DOMAIN_PAGING | \ 80 __IOMMU_DOMAIN_DMA_API) 81 82 struct iommu_domain { 83 unsigned type; 84 const struct iommu_ops *ops; 85 unsigned long pgsize_bitmap; /* Bitmap of page sizes in use */ 86 iommu_fault_handler_t handler; 87 void *handler_token; 88 struct iommu_domain_geometry geometry; 89 void *iova_cookie; 90 }; 91 92 enum iommu_cap { 93 IOMMU_CAP_CACHE_COHERENCY, /* IOMMU can enforce cache coherent DMA 94 transactions */ 95 IOMMU_CAP_INTR_REMAP, /* IOMMU supports interrupt isolation */ 96 IOMMU_CAP_NOEXEC, /* IOMMU_NOEXEC flag */ 97 }; 98 99 /* 100 * Following constraints are specifc to FSL_PAMUV1: 101 * -aperture must be power of 2, and naturally aligned 102 * -number of windows must be power of 2, and address space size 103 * of each window is determined by aperture size / # of windows 104 * -the actual size of the mapped region of a window must be power 105 * of 2 starting with 4KB and physical address must be naturally 106 * aligned. 107 * DOMAIN_ATTR_FSL_PAMUV1 corresponds to the above mentioned contraints. 108 * The caller can invoke iommu_domain_get_attr to check if the underlying 109 * iommu implementation supports these constraints. 110 */ 111 112 enum iommu_attr { 113 DOMAIN_ATTR_GEOMETRY, 114 DOMAIN_ATTR_PAGING, 115 DOMAIN_ATTR_WINDOWS, 116 DOMAIN_ATTR_FSL_PAMU_STASH, 117 DOMAIN_ATTR_FSL_PAMU_ENABLE, 118 DOMAIN_ATTR_FSL_PAMUV1, 119 DOMAIN_ATTR_NESTING, /* two stages of translation */ 120 DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE, 121 DOMAIN_ATTR_IO_PGTABLE_CFG, 122 DOMAIN_ATTR_MAX, 123 }; 124 125 /* These are the possible reserved region types */ 126 enum iommu_resv_type { 127 /* Memory regions which must be mapped 1:1 at all times */ 128 IOMMU_RESV_DIRECT, 129 /* 130 * Memory regions which are advertised to be 1:1 but are 131 * commonly considered relaxable in some conditions, 132 * for instance in device assignment use case (USB, Graphics) 133 */ 134 IOMMU_RESV_DIRECT_RELAXABLE, 135 /* Arbitrary "never map this or give it to a device" address ranges */ 136 IOMMU_RESV_RESERVED, 137 /* Hardware MSI region (untranslated) */ 138 IOMMU_RESV_MSI, 139 /* Software-managed MSI translation window */ 140 IOMMU_RESV_SW_MSI, 141 }; 142 143 /** 144 * struct iommu_resv_region - descriptor for a reserved memory region 145 * @list: Linked list pointers 146 * @start: System physical start address of the region 147 * @length: Length of the region in bytes 148 * @prot: IOMMU Protection flags (READ/WRITE/...) 149 * @type: Type of the reserved region 150 */ 151 struct iommu_resv_region { 152 struct list_head list; 153 phys_addr_t start; 154 size_t length; 155 int prot; 156 enum iommu_resv_type type; 157 }; 158 159 /* Per device IOMMU features */ 160 enum iommu_dev_features { 161 IOMMU_DEV_FEAT_AUX, /* Aux-domain feature */ 162 IOMMU_DEV_FEAT_SVA, /* Shared Virtual Addresses */ 163 }; 164 165 #define IOMMU_PASID_INVALID (-1U) 166 167 #ifdef CONFIG_IOMMU_API 168 169 /** 170 * struct iommu_iotlb_gather - Range information for a pending IOTLB flush 171 * 172 * @start: IOVA representing the start of the range to be flushed 173 * @end: IOVA representing the end of the range to be flushed (exclusive) 174 * @pgsize: The interval at which to perform the flush 175 * 176 * This structure is intended to be updated by multiple calls to the 177 * ->unmap() function in struct iommu_ops before eventually being passed 178 * into ->iotlb_sync(). 179 */ 180 struct iommu_iotlb_gather { 181 unsigned long start; 182 unsigned long end; 183 size_t pgsize; 184 }; 185 186 /** 187 * struct iommu_ops - iommu ops and capabilities 188 * @capable: check capability 189 * @domain_alloc: allocate iommu domain 190 * @domain_free: free iommu domain 191 * @attach_dev: attach device to an iommu domain 192 * @detach_dev: detach device from an iommu domain 193 * @map: map a physically contiguous memory region to an iommu domain 194 * @unmap: unmap a physically contiguous memory region from an iommu domain 195 * @flush_iotlb_all: Synchronously flush all hardware TLBs for this domain 196 * @iotlb_sync_map: Sync mappings created recently using @map to the hardware 197 * @iotlb_sync: Flush all queued ranges from the hardware TLBs and empty flush 198 * queue 199 * @iova_to_phys: translate iova to physical address 200 * @probe_device: Add device to iommu driver handling 201 * @release_device: Remove device from iommu driver handling 202 * @probe_finalize: Do final setup work after the device is added to an IOMMU 203 * group and attached to the groups domain 204 * @device_group: find iommu group for a particular device 205 * @domain_get_attr: Query domain attributes 206 * @domain_set_attr: Change domain attributes 207 * @get_resv_regions: Request list of reserved regions for a device 208 * @put_resv_regions: Free list of reserved regions for a device 209 * @apply_resv_region: Temporary helper call-back for iova reserved ranges 210 * @domain_window_enable: Configure and enable a particular window for a domain 211 * @domain_window_disable: Disable a particular window for a domain 212 * @of_xlate: add OF master IDs to iommu grouping 213 * @is_attach_deferred: Check if domain attach should be deferred from iommu 214 * driver init to device driver init (default no) 215 * @dev_has/enable/disable_feat: per device entries to check/enable/disable 216 * iommu specific features. 217 * @dev_feat_enabled: check enabled feature 218 * @aux_attach/detach_dev: aux-domain specific attach/detach entries. 219 * @aux_get_pasid: get the pasid given an aux-domain 220 * @sva_bind: Bind process address space to device 221 * @sva_unbind: Unbind process address space from device 222 * @sva_get_pasid: Get PASID associated to a SVA handle 223 * @page_response: handle page request response 224 * @cache_invalidate: invalidate translation caches 225 * @sva_bind_gpasid: bind guest pasid and mm 226 * @sva_unbind_gpasid: unbind guest pasid and mm 227 * @def_domain_type: device default domain type, return value: 228 * - IOMMU_DOMAIN_IDENTITY: must use an identity domain 229 * - IOMMU_DOMAIN_DMA: must use a dma domain 230 * - 0: use the default setting 231 * @pgsize_bitmap: bitmap of all possible supported page sizes 232 * @owner: Driver module providing these ops 233 */ 234 struct iommu_ops { 235 bool (*capable)(enum iommu_cap); 236 237 /* Domain allocation and freeing by the iommu driver */ 238 struct iommu_domain *(*domain_alloc)(unsigned iommu_domain_type); 239 void (*domain_free)(struct iommu_domain *); 240 241 int (*attach_dev)(struct iommu_domain *domain, struct device *dev); 242 void (*detach_dev)(struct iommu_domain *domain, struct device *dev); 243 int (*map)(struct iommu_domain *domain, unsigned long iova, 244 phys_addr_t paddr, size_t size, int prot, gfp_t gfp); 245 size_t (*unmap)(struct iommu_domain *domain, unsigned long iova, 246 size_t size, struct iommu_iotlb_gather *iotlb_gather); 247 void (*flush_iotlb_all)(struct iommu_domain *domain); 248 void (*iotlb_sync_map)(struct iommu_domain *domain); 249 void (*iotlb_sync)(struct iommu_domain *domain, 250 struct iommu_iotlb_gather *iotlb_gather); 251 phys_addr_t (*iova_to_phys)(struct iommu_domain *domain, dma_addr_t iova); 252 struct iommu_device *(*probe_device)(struct device *dev); 253 void (*release_device)(struct device *dev); 254 void (*probe_finalize)(struct device *dev); 255 struct iommu_group *(*device_group)(struct device *dev); 256 int (*domain_get_attr)(struct iommu_domain *domain, 257 enum iommu_attr attr, void *data); 258 int (*domain_set_attr)(struct iommu_domain *domain, 259 enum iommu_attr attr, void *data); 260 261 /* Request/Free a list of reserved regions for a device */ 262 void (*get_resv_regions)(struct device *dev, struct list_head *list); 263 void (*put_resv_regions)(struct device *dev, struct list_head *list); 264 void (*apply_resv_region)(struct device *dev, 265 struct iommu_domain *domain, 266 struct iommu_resv_region *region); 267 268 /* Window handling functions */ 269 int (*domain_window_enable)(struct iommu_domain *domain, u32 wnd_nr, 270 phys_addr_t paddr, u64 size, int prot); 271 void (*domain_window_disable)(struct iommu_domain *domain, u32 wnd_nr); 272 273 int (*of_xlate)(struct device *dev, struct of_phandle_args *args); 274 bool (*is_attach_deferred)(struct iommu_domain *domain, struct device *dev); 275 276 /* Per device IOMMU features */ 277 bool (*dev_has_feat)(struct device *dev, enum iommu_dev_features f); 278 bool (*dev_feat_enabled)(struct device *dev, enum iommu_dev_features f); 279 int (*dev_enable_feat)(struct device *dev, enum iommu_dev_features f); 280 int (*dev_disable_feat)(struct device *dev, enum iommu_dev_features f); 281 282 /* Aux-domain specific attach/detach entries */ 283 int (*aux_attach_dev)(struct iommu_domain *domain, struct device *dev); 284 void (*aux_detach_dev)(struct iommu_domain *domain, struct device *dev); 285 int (*aux_get_pasid)(struct iommu_domain *domain, struct device *dev); 286 287 struct iommu_sva *(*sva_bind)(struct device *dev, struct mm_struct *mm, 288 void *drvdata); 289 void (*sva_unbind)(struct iommu_sva *handle); 290 u32 (*sva_get_pasid)(struct iommu_sva *handle); 291 292 int (*page_response)(struct device *dev, 293 struct iommu_fault_event *evt, 294 struct iommu_page_response *msg); 295 int (*cache_invalidate)(struct iommu_domain *domain, struct device *dev, 296 struct iommu_cache_invalidate_info *inv_info); 297 int (*sva_bind_gpasid)(struct iommu_domain *domain, 298 struct device *dev, struct iommu_gpasid_bind_data *data); 299 300 int (*sva_unbind_gpasid)(struct device *dev, u32 pasid); 301 302 int (*def_domain_type)(struct device *dev); 303 304 unsigned long pgsize_bitmap; 305 struct module *owner; 306 }; 307 308 /** 309 * struct iommu_device - IOMMU core representation of one IOMMU hardware 310 * instance 311 * @list: Used by the iommu-core to keep a list of registered iommus 312 * @ops: iommu-ops for talking to this iommu 313 * @dev: struct device for sysfs handling 314 */ 315 struct iommu_device { 316 struct list_head list; 317 const struct iommu_ops *ops; 318 struct fwnode_handle *fwnode; 319 struct device *dev; 320 }; 321 322 /** 323 * struct iommu_fault_event - Generic fault event 324 * 325 * Can represent recoverable faults such as a page requests or 326 * unrecoverable faults such as DMA or IRQ remapping faults. 327 * 328 * @fault: fault descriptor 329 * @list: pending fault event list, used for tracking responses 330 */ 331 struct iommu_fault_event { 332 struct iommu_fault fault; 333 struct list_head list; 334 }; 335 336 /** 337 * struct iommu_fault_param - per-device IOMMU fault data 338 * @handler: Callback function to handle IOMMU faults at device level 339 * @data: handler private data 340 * @faults: holds the pending faults which needs response 341 * @lock: protect pending faults list 342 */ 343 struct iommu_fault_param { 344 iommu_dev_fault_handler_t handler; 345 void *data; 346 struct list_head faults; 347 struct mutex lock; 348 }; 349 350 /** 351 * struct dev_iommu - Collection of per-device IOMMU data 352 * 353 * @fault_param: IOMMU detected device fault reporting data 354 * @fwspec: IOMMU fwspec data 355 * @iommu_dev: IOMMU device this device is linked to 356 * @priv: IOMMU Driver private data 357 * 358 * TODO: migrate other per device data pointers under iommu_dev_data, e.g. 359 * struct iommu_group *iommu_group; 360 */ 361 struct dev_iommu { 362 struct mutex lock; 363 struct iommu_fault_param *fault_param; 364 struct iommu_fwspec *fwspec; 365 struct iommu_device *iommu_dev; 366 void *priv; 367 }; 368 369 int iommu_device_register(struct iommu_device *iommu); 370 void iommu_device_unregister(struct iommu_device *iommu); 371 int iommu_device_sysfs_add(struct iommu_device *iommu, 372 struct device *parent, 373 const struct attribute_group **groups, 374 const char *fmt, ...) __printf(4, 5); 375 void iommu_device_sysfs_remove(struct iommu_device *iommu); 376 int iommu_device_link(struct iommu_device *iommu, struct device *link); 377 void iommu_device_unlink(struct iommu_device *iommu, struct device *link); 378 379 static inline void __iommu_device_set_ops(struct iommu_device *iommu, 380 const struct iommu_ops *ops) 381 { 382 iommu->ops = ops; 383 } 384 385 #define iommu_device_set_ops(iommu, ops) \ 386 do { \ 387 struct iommu_ops *__ops = (struct iommu_ops *)(ops); \ 388 __ops->owner = THIS_MODULE; \ 389 __iommu_device_set_ops(iommu, __ops); \ 390 } while (0) 391 392 static inline void iommu_device_set_fwnode(struct iommu_device *iommu, 393 struct fwnode_handle *fwnode) 394 { 395 iommu->fwnode = fwnode; 396 } 397 398 static inline struct iommu_device *dev_to_iommu_device(struct device *dev) 399 { 400 return (struct iommu_device *)dev_get_drvdata(dev); 401 } 402 403 static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather) 404 { 405 *gather = (struct iommu_iotlb_gather) { 406 .start = ULONG_MAX, 407 }; 408 } 409 410 #define IOMMU_GROUP_NOTIFY_ADD_DEVICE 1 /* Device added */ 411 #define IOMMU_GROUP_NOTIFY_DEL_DEVICE 2 /* Pre Device removed */ 412 #define IOMMU_GROUP_NOTIFY_BIND_DRIVER 3 /* Pre Driver bind */ 413 #define IOMMU_GROUP_NOTIFY_BOUND_DRIVER 4 /* Post Driver bind */ 414 #define IOMMU_GROUP_NOTIFY_UNBIND_DRIVER 5 /* Pre Driver unbind */ 415 #define IOMMU_GROUP_NOTIFY_UNBOUND_DRIVER 6 /* Post Driver unbind */ 416 417 extern int bus_set_iommu(struct bus_type *bus, const struct iommu_ops *ops); 418 extern int bus_iommu_probe(struct bus_type *bus); 419 extern bool iommu_present(struct bus_type *bus); 420 extern bool iommu_capable(struct bus_type *bus, enum iommu_cap cap); 421 extern struct iommu_domain *iommu_domain_alloc(struct bus_type *bus); 422 extern struct iommu_group *iommu_group_get_by_id(int id); 423 extern void iommu_domain_free(struct iommu_domain *domain); 424 extern int iommu_attach_device(struct iommu_domain *domain, 425 struct device *dev); 426 extern void iommu_detach_device(struct iommu_domain *domain, 427 struct device *dev); 428 extern int iommu_uapi_cache_invalidate(struct iommu_domain *domain, 429 struct device *dev, 430 void __user *uinfo); 431 432 extern int iommu_uapi_sva_bind_gpasid(struct iommu_domain *domain, 433 struct device *dev, void __user *udata); 434 extern int iommu_uapi_sva_unbind_gpasid(struct iommu_domain *domain, 435 struct device *dev, void __user *udata); 436 extern int iommu_sva_unbind_gpasid(struct iommu_domain *domain, 437 struct device *dev, ioasid_t pasid); 438 extern struct iommu_domain *iommu_get_domain_for_dev(struct device *dev); 439 extern struct iommu_domain *iommu_get_dma_domain(struct device *dev); 440 extern int iommu_map(struct iommu_domain *domain, unsigned long iova, 441 phys_addr_t paddr, size_t size, int prot); 442 extern int iommu_map_atomic(struct iommu_domain *domain, unsigned long iova, 443 phys_addr_t paddr, size_t size, int prot); 444 extern size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, 445 size_t size); 446 extern size_t iommu_unmap_fast(struct iommu_domain *domain, 447 unsigned long iova, size_t size, 448 struct iommu_iotlb_gather *iotlb_gather); 449 extern size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova, 450 struct scatterlist *sg,unsigned int nents, int prot); 451 extern size_t iommu_map_sg_atomic(struct iommu_domain *domain, 452 unsigned long iova, struct scatterlist *sg, 453 unsigned int nents, int prot); 454 extern phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova); 455 extern void iommu_set_fault_handler(struct iommu_domain *domain, 456 iommu_fault_handler_t handler, void *token); 457 458 extern void iommu_get_resv_regions(struct device *dev, struct list_head *list); 459 extern void iommu_put_resv_regions(struct device *dev, struct list_head *list); 460 extern void generic_iommu_put_resv_regions(struct device *dev, 461 struct list_head *list); 462 extern void iommu_set_default_passthrough(bool cmd_line); 463 extern void iommu_set_default_translated(bool cmd_line); 464 extern bool iommu_default_passthrough(void); 465 extern struct iommu_resv_region * 466 iommu_alloc_resv_region(phys_addr_t start, size_t length, int prot, 467 enum iommu_resv_type type); 468 extern int iommu_get_group_resv_regions(struct iommu_group *group, 469 struct list_head *head); 470 471 extern int iommu_attach_group(struct iommu_domain *domain, 472 struct iommu_group *group); 473 extern void iommu_detach_group(struct iommu_domain *domain, 474 struct iommu_group *group); 475 extern struct iommu_group *iommu_group_alloc(void); 476 extern void *iommu_group_get_iommudata(struct iommu_group *group); 477 extern void iommu_group_set_iommudata(struct iommu_group *group, 478 void *iommu_data, 479 void (*release)(void *iommu_data)); 480 extern int iommu_group_set_name(struct iommu_group *group, const char *name); 481 extern int iommu_group_add_device(struct iommu_group *group, 482 struct device *dev); 483 extern void iommu_group_remove_device(struct device *dev); 484 extern int iommu_group_for_each_dev(struct iommu_group *group, void *data, 485 int (*fn)(struct device *, void *)); 486 extern struct iommu_group *iommu_group_get(struct device *dev); 487 extern struct iommu_group *iommu_group_ref_get(struct iommu_group *group); 488 extern void iommu_group_put(struct iommu_group *group); 489 extern int iommu_group_register_notifier(struct iommu_group *group, 490 struct notifier_block *nb); 491 extern int iommu_group_unregister_notifier(struct iommu_group *group, 492 struct notifier_block *nb); 493 extern int iommu_register_device_fault_handler(struct device *dev, 494 iommu_dev_fault_handler_t handler, 495 void *data); 496 497 extern int iommu_unregister_device_fault_handler(struct device *dev); 498 499 extern int iommu_report_device_fault(struct device *dev, 500 struct iommu_fault_event *evt); 501 extern int iommu_page_response(struct device *dev, 502 struct iommu_page_response *msg); 503 504 extern int iommu_group_id(struct iommu_group *group); 505 extern struct iommu_domain *iommu_group_default_domain(struct iommu_group *); 506 507 extern int iommu_domain_get_attr(struct iommu_domain *domain, enum iommu_attr, 508 void *data); 509 extern int iommu_domain_set_attr(struct iommu_domain *domain, enum iommu_attr, 510 void *data); 511 512 /* Window handling function prototypes */ 513 extern int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr, 514 phys_addr_t offset, u64 size, 515 int prot); 516 extern void iommu_domain_window_disable(struct iommu_domain *domain, u32 wnd_nr); 517 518 extern int report_iommu_fault(struct iommu_domain *domain, struct device *dev, 519 unsigned long iova, int flags); 520 521 static inline void iommu_flush_iotlb_all(struct iommu_domain *domain) 522 { 523 if (domain->ops->flush_iotlb_all) 524 domain->ops->flush_iotlb_all(domain); 525 } 526 527 static inline void iommu_iotlb_sync(struct iommu_domain *domain, 528 struct iommu_iotlb_gather *iotlb_gather) 529 { 530 if (domain->ops->iotlb_sync) 531 domain->ops->iotlb_sync(domain, iotlb_gather); 532 533 iommu_iotlb_gather_init(iotlb_gather); 534 } 535 536 static inline void iommu_iotlb_gather_add_page(struct iommu_domain *domain, 537 struct iommu_iotlb_gather *gather, 538 unsigned long iova, size_t size) 539 { 540 unsigned long start = iova, end = start + size; 541 542 /* 543 * If the new page is disjoint from the current range or is mapped at 544 * a different granularity, then sync the TLB so that the gather 545 * structure can be rewritten. 546 */ 547 if (gather->pgsize != size || 548 end < gather->start || start > gather->end) { 549 if (gather->pgsize) 550 iommu_iotlb_sync(domain, gather); 551 gather->pgsize = size; 552 } 553 554 if (gather->end < end) 555 gather->end = end; 556 557 if (gather->start > start) 558 gather->start = start; 559 } 560 561 /* PCI device grouping function */ 562 extern struct iommu_group *pci_device_group(struct device *dev); 563 /* Generic device grouping function */ 564 extern struct iommu_group *generic_device_group(struct device *dev); 565 /* FSL-MC device grouping function */ 566 struct iommu_group *fsl_mc_device_group(struct device *dev); 567 568 /** 569 * struct iommu_fwspec - per-device IOMMU instance data 570 * @ops: ops for this device's IOMMU 571 * @iommu_fwnode: firmware handle for this device's IOMMU 572 * @iommu_priv: IOMMU driver private data for this device 573 * @num_pasid_bits: number of PASID bits supported by this device 574 * @num_ids: number of associated device IDs 575 * @ids: IDs which this device may present to the IOMMU 576 */ 577 struct iommu_fwspec { 578 const struct iommu_ops *ops; 579 struct fwnode_handle *iommu_fwnode; 580 u32 flags; 581 u32 num_pasid_bits; 582 unsigned int num_ids; 583 u32 ids[]; 584 }; 585 586 /* ATS is supported */ 587 #define IOMMU_FWSPEC_PCI_RC_ATS (1 << 0) 588 589 /** 590 * struct iommu_sva - handle to a device-mm bond 591 */ 592 struct iommu_sva { 593 struct device *dev; 594 }; 595 596 int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode, 597 const struct iommu_ops *ops); 598 void iommu_fwspec_free(struct device *dev); 599 int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids); 600 const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode); 601 602 static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev) 603 { 604 if (dev->iommu) 605 return dev->iommu->fwspec; 606 else 607 return NULL; 608 } 609 610 static inline void dev_iommu_fwspec_set(struct device *dev, 611 struct iommu_fwspec *fwspec) 612 { 613 dev->iommu->fwspec = fwspec; 614 } 615 616 static inline void *dev_iommu_priv_get(struct device *dev) 617 { 618 return dev->iommu->priv; 619 } 620 621 static inline void dev_iommu_priv_set(struct device *dev, void *priv) 622 { 623 dev->iommu->priv = priv; 624 } 625 626 int iommu_probe_device(struct device *dev); 627 void iommu_release_device(struct device *dev); 628 629 bool iommu_dev_has_feature(struct device *dev, enum iommu_dev_features f); 630 int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features f); 631 int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features f); 632 bool iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features f); 633 int iommu_aux_attach_device(struct iommu_domain *domain, struct device *dev); 634 void iommu_aux_detach_device(struct iommu_domain *domain, struct device *dev); 635 int iommu_aux_get_pasid(struct iommu_domain *domain, struct device *dev); 636 637 struct iommu_sva *iommu_sva_bind_device(struct device *dev, 638 struct mm_struct *mm, 639 void *drvdata); 640 void iommu_sva_unbind_device(struct iommu_sva *handle); 641 u32 iommu_sva_get_pasid(struct iommu_sva *handle); 642 643 #else /* CONFIG_IOMMU_API */ 644 645 struct iommu_ops {}; 646 struct iommu_group {}; 647 struct iommu_fwspec {}; 648 struct iommu_device {}; 649 struct iommu_fault_param {}; 650 struct iommu_iotlb_gather {}; 651 652 static inline bool iommu_present(struct bus_type *bus) 653 { 654 return false; 655 } 656 657 static inline bool iommu_capable(struct bus_type *bus, enum iommu_cap cap) 658 { 659 return false; 660 } 661 662 static inline struct iommu_domain *iommu_domain_alloc(struct bus_type *bus) 663 { 664 return NULL; 665 } 666 667 static inline struct iommu_group *iommu_group_get_by_id(int id) 668 { 669 return NULL; 670 } 671 672 static inline void iommu_domain_free(struct iommu_domain *domain) 673 { 674 } 675 676 static inline int iommu_attach_device(struct iommu_domain *domain, 677 struct device *dev) 678 { 679 return -ENODEV; 680 } 681 682 static inline void iommu_detach_device(struct iommu_domain *domain, 683 struct device *dev) 684 { 685 } 686 687 static inline struct iommu_domain *iommu_get_domain_for_dev(struct device *dev) 688 { 689 return NULL; 690 } 691 692 static inline int iommu_map(struct iommu_domain *domain, unsigned long iova, 693 phys_addr_t paddr, size_t size, int prot) 694 { 695 return -ENODEV; 696 } 697 698 static inline int iommu_map_atomic(struct iommu_domain *domain, 699 unsigned long iova, phys_addr_t paddr, 700 size_t size, int prot) 701 { 702 return -ENODEV; 703 } 704 705 static inline size_t iommu_unmap(struct iommu_domain *domain, 706 unsigned long iova, size_t size) 707 { 708 return 0; 709 } 710 711 static inline size_t iommu_unmap_fast(struct iommu_domain *domain, 712 unsigned long iova, int gfp_order, 713 struct iommu_iotlb_gather *iotlb_gather) 714 { 715 return 0; 716 } 717 718 static inline size_t iommu_map_sg(struct iommu_domain *domain, 719 unsigned long iova, struct scatterlist *sg, 720 unsigned int nents, int prot) 721 { 722 return 0; 723 } 724 725 static inline size_t iommu_map_sg_atomic(struct iommu_domain *domain, 726 unsigned long iova, struct scatterlist *sg, 727 unsigned int nents, int prot) 728 { 729 return 0; 730 } 731 732 static inline void iommu_flush_iotlb_all(struct iommu_domain *domain) 733 { 734 } 735 736 static inline void iommu_iotlb_sync(struct iommu_domain *domain, 737 struct iommu_iotlb_gather *iotlb_gather) 738 { 739 } 740 741 static inline int iommu_domain_window_enable(struct iommu_domain *domain, 742 u32 wnd_nr, phys_addr_t paddr, 743 u64 size, int prot) 744 { 745 return -ENODEV; 746 } 747 748 static inline void iommu_domain_window_disable(struct iommu_domain *domain, 749 u32 wnd_nr) 750 { 751 } 752 753 static inline phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) 754 { 755 return 0; 756 } 757 758 static inline void iommu_set_fault_handler(struct iommu_domain *domain, 759 iommu_fault_handler_t handler, void *token) 760 { 761 } 762 763 static inline void iommu_get_resv_regions(struct device *dev, 764 struct list_head *list) 765 { 766 } 767 768 static inline void iommu_put_resv_regions(struct device *dev, 769 struct list_head *list) 770 { 771 } 772 773 static inline int iommu_get_group_resv_regions(struct iommu_group *group, 774 struct list_head *head) 775 { 776 return -ENODEV; 777 } 778 779 static inline void iommu_set_default_passthrough(bool cmd_line) 780 { 781 } 782 783 static inline void iommu_set_default_translated(bool cmd_line) 784 { 785 } 786 787 static inline bool iommu_default_passthrough(void) 788 { 789 return true; 790 } 791 792 static inline int iommu_attach_group(struct iommu_domain *domain, 793 struct iommu_group *group) 794 { 795 return -ENODEV; 796 } 797 798 static inline void iommu_detach_group(struct iommu_domain *domain, 799 struct iommu_group *group) 800 { 801 } 802 803 static inline struct iommu_group *iommu_group_alloc(void) 804 { 805 return ERR_PTR(-ENODEV); 806 } 807 808 static inline void *iommu_group_get_iommudata(struct iommu_group *group) 809 { 810 return NULL; 811 } 812 813 static inline void iommu_group_set_iommudata(struct iommu_group *group, 814 void *iommu_data, 815 void (*release)(void *iommu_data)) 816 { 817 } 818 819 static inline int iommu_group_set_name(struct iommu_group *group, 820 const char *name) 821 { 822 return -ENODEV; 823 } 824 825 static inline int iommu_group_add_device(struct iommu_group *group, 826 struct device *dev) 827 { 828 return -ENODEV; 829 } 830 831 static inline void iommu_group_remove_device(struct device *dev) 832 { 833 } 834 835 static inline int iommu_group_for_each_dev(struct iommu_group *group, 836 void *data, 837 int (*fn)(struct device *, void *)) 838 { 839 return -ENODEV; 840 } 841 842 static inline struct iommu_group *iommu_group_get(struct device *dev) 843 { 844 return NULL; 845 } 846 847 static inline void iommu_group_put(struct iommu_group *group) 848 { 849 } 850 851 static inline int iommu_group_register_notifier(struct iommu_group *group, 852 struct notifier_block *nb) 853 { 854 return -ENODEV; 855 } 856 857 static inline int iommu_group_unregister_notifier(struct iommu_group *group, 858 struct notifier_block *nb) 859 { 860 return 0; 861 } 862 863 static inline 864 int iommu_register_device_fault_handler(struct device *dev, 865 iommu_dev_fault_handler_t handler, 866 void *data) 867 { 868 return -ENODEV; 869 } 870 871 static inline int iommu_unregister_device_fault_handler(struct device *dev) 872 { 873 return 0; 874 } 875 876 static inline 877 int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt) 878 { 879 return -ENODEV; 880 } 881 882 static inline int iommu_page_response(struct device *dev, 883 struct iommu_page_response *msg) 884 { 885 return -ENODEV; 886 } 887 888 static inline int iommu_group_id(struct iommu_group *group) 889 { 890 return -ENODEV; 891 } 892 893 static inline int iommu_domain_get_attr(struct iommu_domain *domain, 894 enum iommu_attr attr, void *data) 895 { 896 return -EINVAL; 897 } 898 899 static inline int iommu_domain_set_attr(struct iommu_domain *domain, 900 enum iommu_attr attr, void *data) 901 { 902 return -EINVAL; 903 } 904 905 static inline int iommu_device_register(struct iommu_device *iommu) 906 { 907 return -ENODEV; 908 } 909 910 static inline void iommu_device_set_ops(struct iommu_device *iommu, 911 const struct iommu_ops *ops) 912 { 913 } 914 915 static inline void iommu_device_set_fwnode(struct iommu_device *iommu, 916 struct fwnode_handle *fwnode) 917 { 918 } 919 920 static inline struct iommu_device *dev_to_iommu_device(struct device *dev) 921 { 922 return NULL; 923 } 924 925 static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather) 926 { 927 } 928 929 static inline void iommu_iotlb_gather_add_page(struct iommu_domain *domain, 930 struct iommu_iotlb_gather *gather, 931 unsigned long iova, size_t size) 932 { 933 } 934 935 static inline void iommu_device_unregister(struct iommu_device *iommu) 936 { 937 } 938 939 static inline int iommu_device_sysfs_add(struct iommu_device *iommu, 940 struct device *parent, 941 const struct attribute_group **groups, 942 const char *fmt, ...) 943 { 944 return -ENODEV; 945 } 946 947 static inline void iommu_device_sysfs_remove(struct iommu_device *iommu) 948 { 949 } 950 951 static inline int iommu_device_link(struct device *dev, struct device *link) 952 { 953 return -EINVAL; 954 } 955 956 static inline void iommu_device_unlink(struct device *dev, struct device *link) 957 { 958 } 959 960 static inline int iommu_fwspec_init(struct device *dev, 961 struct fwnode_handle *iommu_fwnode, 962 const struct iommu_ops *ops) 963 { 964 return -ENODEV; 965 } 966 967 static inline void iommu_fwspec_free(struct device *dev) 968 { 969 } 970 971 static inline int iommu_fwspec_add_ids(struct device *dev, u32 *ids, 972 int num_ids) 973 { 974 return -ENODEV; 975 } 976 977 static inline 978 const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode) 979 { 980 return NULL; 981 } 982 983 static inline bool 984 iommu_dev_has_feature(struct device *dev, enum iommu_dev_features feat) 985 { 986 return false; 987 } 988 989 static inline bool 990 iommu_dev_feature_enabled(struct device *dev, enum iommu_dev_features feat) 991 { 992 return false; 993 } 994 995 static inline int 996 iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features feat) 997 { 998 return -ENODEV; 999 } 1000 1001 static inline int 1002 iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat) 1003 { 1004 return -ENODEV; 1005 } 1006 1007 static inline int 1008 iommu_aux_attach_device(struct iommu_domain *domain, struct device *dev) 1009 { 1010 return -ENODEV; 1011 } 1012 1013 static inline void 1014 iommu_aux_detach_device(struct iommu_domain *domain, struct device *dev) 1015 { 1016 } 1017 1018 static inline int 1019 iommu_aux_get_pasid(struct iommu_domain *domain, struct device *dev) 1020 { 1021 return -ENODEV; 1022 } 1023 1024 static inline struct iommu_sva * 1025 iommu_sva_bind_device(struct device *dev, struct mm_struct *mm, void *drvdata) 1026 { 1027 return NULL; 1028 } 1029 1030 static inline void iommu_sva_unbind_device(struct iommu_sva *handle) 1031 { 1032 } 1033 1034 static inline u32 iommu_sva_get_pasid(struct iommu_sva *handle) 1035 { 1036 return IOMMU_PASID_INVALID; 1037 } 1038 1039 static inline int 1040 iommu_uapi_cache_invalidate(struct iommu_domain *domain, 1041 struct device *dev, 1042 struct iommu_cache_invalidate_info *inv_info) 1043 { 1044 return -ENODEV; 1045 } 1046 1047 static inline int iommu_uapi_sva_bind_gpasid(struct iommu_domain *domain, 1048 struct device *dev, void __user *udata) 1049 { 1050 return -ENODEV; 1051 } 1052 1053 static inline int iommu_uapi_sva_unbind_gpasid(struct iommu_domain *domain, 1054 struct device *dev, void __user *udata) 1055 { 1056 return -ENODEV; 1057 } 1058 1059 static inline int iommu_sva_unbind_gpasid(struct iommu_domain *domain, 1060 struct device *dev, 1061 ioasid_t pasid) 1062 { 1063 return -ENODEV; 1064 } 1065 1066 static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev) 1067 { 1068 return NULL; 1069 } 1070 #endif /* CONFIG_IOMMU_API */ 1071 1072 /** 1073 * iommu_map_sgtable - Map the given buffer to the IOMMU domain 1074 * @domain: The IOMMU domain to perform the mapping 1075 * @iova: The start address to map the buffer 1076 * @sgt: The sg_table object describing the buffer 1077 * @prot: IOMMU protection bits 1078 * 1079 * Creates a mapping at @iova for the buffer described by a scatterlist 1080 * stored in the given sg_table object in the provided IOMMU domain. 1081 */ 1082 static inline size_t iommu_map_sgtable(struct iommu_domain *domain, 1083 unsigned long iova, struct sg_table *sgt, int prot) 1084 { 1085 return iommu_map_sg(domain, iova, sgt->sgl, sgt->orig_nents, prot); 1086 } 1087 1088 #ifdef CONFIG_IOMMU_DEBUGFS 1089 extern struct dentry *iommu_debugfs_dir; 1090 void iommu_debugfs_setup(void); 1091 #else 1092 static inline void iommu_debugfs_setup(void) {} 1093 #endif 1094 1095 #endif /* __LINUX_IOMMU_H */ 1096