1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_SCATTERLIST_H 3 #define _LINUX_SCATTERLIST_H 4 5 #include <linux/string.h> 6 #include <linux/types.h> 7 #include <linux/bug.h> 8 #include <linux/mm.h> 9 #include <asm/io.h> 10 11 struct scatterlist { 12 #ifdef CONFIG_DEBUG_SG 13 unsigned long sg_magic; 14 #endif 15 unsigned long page_link; 16 unsigned int offset; 17 unsigned int length; 18 dma_addr_t dma_address; 19 #ifdef CONFIG_NEED_SG_DMA_LENGTH 20 unsigned int dma_length; 21 #endif 22 }; 23 24 /* 25 * Since the above length field is an unsigned int, below we define the maximum 26 * length in bytes that can be stored in one scatterlist entry. 27 */ 28 #define SCATTERLIST_MAX_SEGMENT (UINT_MAX & PAGE_MASK) 29 30 /* 31 * These macros should be used after a dma_map_sg call has been done 32 * to get bus addresses of each of the SG entries and their lengths. 33 * You should only work with the number of sg entries dma_map_sg 34 * returns, or alternatively stop on the first sg_dma_len(sg) which 35 * is 0. 36 */ 37 #define sg_dma_address(sg) ((sg)->dma_address) 38 39 #ifdef CONFIG_NEED_SG_DMA_LENGTH 40 #define sg_dma_len(sg) ((sg)->dma_length) 41 #else 42 #define sg_dma_len(sg) ((sg)->length) 43 #endif 44 45 struct sg_table { 46 struct scatterlist *sgl; /* the list */ 47 unsigned int nents; /* number of mapped entries */ 48 unsigned int orig_nents; /* original size of list */ 49 }; 50 51 /* 52 * Notes on SG table design. 53 * 54 * We use the unsigned long page_link field in the scatterlist struct to place 55 * the page pointer AND encode information about the sg table as well. The two 56 * lower bits are reserved for this information. 57 * 58 * If bit 0 is set, then the page_link contains a pointer to the next sg 59 * table list. Otherwise the next entry is at sg + 1. 60 * 61 * If bit 1 is set, then this sg entry is the last element in a list. 62 * 63 * See sg_next(). 64 * 65 */ 66 67 #define SG_MAGIC 0x87654321 68 69 /* 70 * We overload the LSB of the page pointer to indicate whether it's 71 * a valid sg entry, or whether it points to the start of a new scatterlist. 72 * Those low bits are there for everyone! (thanks mason :-) 73 */ 74 #define sg_is_chain(sg) ((sg)->page_link & 0x01) 75 #define sg_is_last(sg) ((sg)->page_link & 0x02) 76 #define sg_chain_ptr(sg) \ 77 ((struct scatterlist *) ((sg)->page_link & ~0x03)) 78 79 /** 80 * sg_assign_page - Assign a given page to an SG entry 81 * @sg: SG entry 82 * @page: The page 83 * 84 * Description: 85 * Assign page to sg entry. Also see sg_set_page(), the most commonly used 86 * variant. 87 * 88 **/ 89 static inline void sg_assign_page(struct scatterlist *sg, struct page *page) 90 { 91 unsigned long page_link = sg->page_link & 0x3; 92 93 /* 94 * In order for the low bit stealing approach to work, pages 95 * must be aligned at a 32-bit boundary as a minimum. 96 */ 97 BUG_ON((unsigned long) page & 0x03); 98 #ifdef CONFIG_DEBUG_SG 99 BUG_ON(sg->sg_magic != SG_MAGIC); 100 BUG_ON(sg_is_chain(sg)); 101 #endif 102 sg->page_link = page_link | (unsigned long) page; 103 } 104 105 /** 106 * sg_set_page - Set sg entry to point at given page 107 * @sg: SG entry 108 * @page: The page 109 * @len: Length of data 110 * @offset: Offset into page 111 * 112 * Description: 113 * Use this function to set an sg entry pointing at a page, never assign 114 * the page directly. We encode sg table information in the lower bits 115 * of the page pointer. See sg_page() for looking up the page belonging 116 * to an sg entry. 117 * 118 **/ 119 static inline void sg_set_page(struct scatterlist *sg, struct page *page, 120 unsigned int len, unsigned int offset) 121 { 122 sg_assign_page(sg, page); 123 sg->offset = offset; 124 sg->length = len; 125 } 126 127 static inline struct page *sg_page(struct scatterlist *sg) 128 { 129 #ifdef CONFIG_DEBUG_SG 130 BUG_ON(sg->sg_magic != SG_MAGIC); 131 BUG_ON(sg_is_chain(sg)); 132 #endif 133 return (struct page *)((sg)->page_link & ~0x3); 134 } 135 136 /** 137 * sg_set_buf - Set sg entry to point at given data 138 * @sg: SG entry 139 * @buf: Data 140 * @buflen: Data length 141 * 142 **/ 143 static inline void sg_set_buf(struct scatterlist *sg, const void *buf, 144 unsigned int buflen) 145 { 146 #ifdef CONFIG_DEBUG_SG 147 BUG_ON(!virt_addr_valid(buf)); 148 #endif 149 sg_set_page(sg, virt_to_page(buf), buflen, offset_in_page(buf)); 150 } 151 152 /* 153 * Loop over each sg element, following the pointer to a new list if necessary 154 */ 155 #define for_each_sg(sglist, sg, nr, __i) \ 156 for (__i = 0, sg = (sglist); __i < (nr); __i++, sg = sg_next(sg)) 157 158 /** 159 * sg_chain - Chain two sglists together 160 * @prv: First scatterlist 161 * @prv_nents: Number of entries in prv 162 * @sgl: Second scatterlist 163 * 164 * Description: 165 * Links @prv@ and @sgl@ together, to form a longer scatterlist. 166 * 167 **/ 168 static inline void sg_chain(struct scatterlist *prv, unsigned int prv_nents, 169 struct scatterlist *sgl) 170 { 171 /* 172 * offset and length are unused for chain entry. Clear them. 173 */ 174 prv[prv_nents - 1].offset = 0; 175 prv[prv_nents - 1].length = 0; 176 177 /* 178 * Set lowest bit to indicate a link pointer, and make sure to clear 179 * the termination bit if it happens to be set. 180 */ 181 prv[prv_nents - 1].page_link = ((unsigned long) sgl | 0x01) & ~0x02; 182 } 183 184 /** 185 * sg_mark_end - Mark the end of the scatterlist 186 * @sg: SG entryScatterlist 187 * 188 * Description: 189 * Marks the passed in sg entry as the termination point for the sg 190 * table. A call to sg_next() on this entry will return NULL. 191 * 192 **/ 193 static inline void sg_mark_end(struct scatterlist *sg) 194 { 195 #ifdef CONFIG_DEBUG_SG 196 BUG_ON(sg->sg_magic != SG_MAGIC); 197 #endif 198 /* 199 * Set termination bit, clear potential chain bit 200 */ 201 sg->page_link |= 0x02; 202 sg->page_link &= ~0x01; 203 } 204 205 /** 206 * sg_unmark_end - Undo setting the end of the scatterlist 207 * @sg: SG entryScatterlist 208 * 209 * Description: 210 * Removes the termination marker from the given entry of the scatterlist. 211 * 212 **/ 213 static inline void sg_unmark_end(struct scatterlist *sg) 214 { 215 #ifdef CONFIG_DEBUG_SG 216 BUG_ON(sg->sg_magic != SG_MAGIC); 217 #endif 218 sg->page_link &= ~0x02; 219 } 220 221 /** 222 * sg_phys - Return physical address of an sg entry 223 * @sg: SG entry 224 * 225 * Description: 226 * This calls page_to_phys() on the page in this sg entry, and adds the 227 * sg offset. The caller must know that it is legal to call page_to_phys() 228 * on the sg page. 229 * 230 **/ 231 static inline dma_addr_t sg_phys(struct scatterlist *sg) 232 { 233 return page_to_phys(sg_page(sg)) + sg->offset; 234 } 235 236 /** 237 * sg_virt - Return virtual address of an sg entry 238 * @sg: SG entry 239 * 240 * Description: 241 * This calls page_address() on the page in this sg entry, and adds the 242 * sg offset. The caller must know that the sg page has a valid virtual 243 * mapping. 244 * 245 **/ 246 static inline void *sg_virt(struct scatterlist *sg) 247 { 248 return page_address(sg_page(sg)) + sg->offset; 249 } 250 251 int sg_nents(struct scatterlist *sg); 252 int sg_nents_for_len(struct scatterlist *sg, u64 len); 253 struct scatterlist *sg_next(struct scatterlist *); 254 struct scatterlist *sg_last(struct scatterlist *s, unsigned int); 255 void sg_init_table(struct scatterlist *, unsigned int); 256 void sg_init_one(struct scatterlist *, const void *, unsigned int); 257 int sg_split(struct scatterlist *in, const int in_mapped_nents, 258 const off_t skip, const int nb_splits, 259 const size_t *split_sizes, 260 struct scatterlist **out, int *out_mapped_nents, 261 gfp_t gfp_mask); 262 263 typedef struct scatterlist *(sg_alloc_fn)(unsigned int, gfp_t); 264 typedef void (sg_free_fn)(struct scatterlist *, unsigned int); 265 266 void __sg_free_table(struct sg_table *, unsigned int, bool, sg_free_fn *); 267 void sg_free_table(struct sg_table *); 268 int __sg_alloc_table(struct sg_table *, unsigned int, unsigned int, 269 struct scatterlist *, gfp_t, sg_alloc_fn *); 270 int sg_alloc_table(struct sg_table *, unsigned int, gfp_t); 271 int __sg_alloc_table_from_pages(struct sg_table *sgt, struct page **pages, 272 unsigned int n_pages, unsigned int offset, 273 unsigned long size, unsigned int max_segment, 274 gfp_t gfp_mask); 275 int sg_alloc_table_from_pages(struct sg_table *sgt, struct page **pages, 276 unsigned int n_pages, unsigned int offset, 277 unsigned long size, gfp_t gfp_mask); 278 279 size_t sg_copy_buffer(struct scatterlist *sgl, unsigned int nents, void *buf, 280 size_t buflen, off_t skip, bool to_buffer); 281 282 size_t sg_copy_from_buffer(struct scatterlist *sgl, unsigned int nents, 283 const void *buf, size_t buflen); 284 size_t sg_copy_to_buffer(struct scatterlist *sgl, unsigned int nents, 285 void *buf, size_t buflen); 286 287 size_t sg_pcopy_from_buffer(struct scatterlist *sgl, unsigned int nents, 288 const void *buf, size_t buflen, off_t skip); 289 size_t sg_pcopy_to_buffer(struct scatterlist *sgl, unsigned int nents, 290 void *buf, size_t buflen, off_t skip); 291 size_t sg_zero_buffer(struct scatterlist *sgl, unsigned int nents, 292 size_t buflen, off_t skip); 293 294 /* 295 * Maximum number of entries that will be allocated in one piece, if 296 * a list larger than this is required then chaining will be utilized. 297 */ 298 #define SG_MAX_SINGLE_ALLOC (PAGE_SIZE / sizeof(struct scatterlist)) 299 300 /* 301 * The maximum number of SG segments that we will put inside a 302 * scatterlist (unless chaining is used). Should ideally fit inside a 303 * single page, to avoid a higher order allocation. We could define this 304 * to SG_MAX_SINGLE_ALLOC to pack correctly at the highest order. The 305 * minimum value is 32 306 */ 307 #define SG_CHUNK_SIZE 128 308 309 /* 310 * Like SG_CHUNK_SIZE, but for archs that have sg chaining. This limit 311 * is totally arbitrary, a setting of 2048 will get you at least 8mb ios. 312 */ 313 #ifdef CONFIG_ARCH_HAS_SG_CHAIN 314 #define SG_MAX_SEGMENTS 2048 315 #else 316 #define SG_MAX_SEGMENTS SG_CHUNK_SIZE 317 #endif 318 319 #ifdef CONFIG_SG_POOL 320 void sg_free_table_chained(struct sg_table *table, bool first_chunk); 321 int sg_alloc_table_chained(struct sg_table *table, int nents, 322 struct scatterlist *first_chunk); 323 #endif 324 325 /* 326 * sg page iterator 327 * 328 * Iterates over sg entries page-by-page. On each successful iteration, 329 * you can call sg_page_iter_page(@piter) and sg_page_iter_dma_address(@piter) 330 * to get the current page and its dma address. @piter->sg will point to the 331 * sg holding this page and @piter->sg_pgoffset to the page's page offset 332 * within the sg. The iteration will stop either when a maximum number of sg 333 * entries was reached or a terminating sg (sg_last(sg) == true) was reached. 334 */ 335 struct sg_page_iter { 336 struct scatterlist *sg; /* sg holding the page */ 337 unsigned int sg_pgoffset; /* page offset within the sg */ 338 339 /* these are internal states, keep away */ 340 unsigned int __nents; /* remaining sg entries */ 341 int __pg_advance; /* nr pages to advance at the 342 * next step */ 343 }; 344 345 bool __sg_page_iter_next(struct sg_page_iter *piter); 346 void __sg_page_iter_start(struct sg_page_iter *piter, 347 struct scatterlist *sglist, unsigned int nents, 348 unsigned long pgoffset); 349 /** 350 * sg_page_iter_page - get the current page held by the page iterator 351 * @piter: page iterator holding the page 352 */ 353 static inline struct page *sg_page_iter_page(struct sg_page_iter *piter) 354 { 355 return nth_page(sg_page(piter->sg), piter->sg_pgoffset); 356 } 357 358 /** 359 * sg_page_iter_dma_address - get the dma address of the current page held by 360 * the page iterator. 361 * @piter: page iterator holding the page 362 */ 363 static inline dma_addr_t sg_page_iter_dma_address(struct sg_page_iter *piter) 364 { 365 return sg_dma_address(piter->sg) + (piter->sg_pgoffset << PAGE_SHIFT); 366 } 367 368 /** 369 * for_each_sg_page - iterate over the pages of the given sg list 370 * @sglist: sglist to iterate over 371 * @piter: page iterator to hold current page, sg, sg_pgoffset 372 * @nents: maximum number of sg entries to iterate over 373 * @pgoffset: starting page offset 374 */ 375 #define for_each_sg_page(sglist, piter, nents, pgoffset) \ 376 for (__sg_page_iter_start((piter), (sglist), (nents), (pgoffset)); \ 377 __sg_page_iter_next(piter);) 378 379 /* 380 * Mapping sg iterator 381 * 382 * Iterates over sg entries mapping page-by-page. On each successful 383 * iteration, @miter->page points to the mapped page and 384 * @miter->length bytes of data can be accessed at @miter->addr. As 385 * long as an interation is enclosed between start and stop, the user 386 * is free to choose control structure and when to stop. 387 * 388 * @miter->consumed is set to @miter->length on each iteration. It 389 * can be adjusted if the user can't consume all the bytes in one go. 390 * Also, a stopped iteration can be resumed by calling next on it. 391 * This is useful when iteration needs to release all resources and 392 * continue later (e.g. at the next interrupt). 393 */ 394 395 #define SG_MITER_ATOMIC (1 << 0) /* use kmap_atomic */ 396 #define SG_MITER_TO_SG (1 << 1) /* flush back to phys on unmap */ 397 #define SG_MITER_FROM_SG (1 << 2) /* nop */ 398 399 struct sg_mapping_iter { 400 /* the following three fields can be accessed directly */ 401 struct page *page; /* currently mapped page */ 402 void *addr; /* pointer to the mapped area */ 403 size_t length; /* length of the mapped area */ 404 size_t consumed; /* number of consumed bytes */ 405 struct sg_page_iter piter; /* page iterator */ 406 407 /* these are internal states, keep away */ 408 unsigned int __offset; /* offset within page */ 409 unsigned int __remaining; /* remaining bytes on page */ 410 unsigned int __flags; 411 }; 412 413 void sg_miter_start(struct sg_mapping_iter *miter, struct scatterlist *sgl, 414 unsigned int nents, unsigned int flags); 415 bool sg_miter_skip(struct sg_mapping_iter *miter, off_t offset); 416 bool sg_miter_next(struct sg_mapping_iter *miter); 417 void sg_miter_stop(struct sg_mapping_iter *miter); 418 419 #endif /* _LINUX_SCATTERLIST_H */ 420