1 #ifndef _LINUX_SCATTERLIST_H 2 #define _LINUX_SCATTERLIST_H 3 4 #include <linux/string.h> 5 #include <linux/types.h> 6 #include <linux/bug.h> 7 #include <linux/mm.h> 8 #include <asm/io.h> 9 10 struct scatterlist { 11 #ifdef CONFIG_DEBUG_SG 12 unsigned long sg_magic; 13 #endif 14 unsigned long page_link; 15 unsigned int offset; 16 unsigned int length; 17 dma_addr_t dma_address; 18 #ifdef CONFIG_NEED_SG_DMA_LENGTH 19 unsigned int dma_length; 20 #endif 21 }; 22 23 /* 24 * These macros should be used after a dma_map_sg call has been done 25 * to get bus addresses of each of the SG entries and their lengths. 26 * You should only work with the number of sg entries dma_map_sg 27 * returns, or alternatively stop on the first sg_dma_len(sg) which 28 * is 0. 29 */ 30 #define sg_dma_address(sg) ((sg)->dma_address) 31 32 #ifdef CONFIG_NEED_SG_DMA_LENGTH 33 #define sg_dma_len(sg) ((sg)->dma_length) 34 #else 35 #define sg_dma_len(sg) ((sg)->length) 36 #endif 37 38 struct sg_table { 39 struct scatterlist *sgl; /* the list */ 40 unsigned int nents; /* number of mapped entries */ 41 unsigned int orig_nents; /* original size of list */ 42 }; 43 44 /* 45 * Notes on SG table design. 46 * 47 * We use the unsigned long page_link field in the scatterlist struct to place 48 * the page pointer AND encode information about the sg table as well. The two 49 * lower bits are reserved for this information. 50 * 51 * If bit 0 is set, then the page_link contains a pointer to the next sg 52 * table list. Otherwise the next entry is at sg + 1. 53 * 54 * If bit 1 is set, then this sg entry is the last element in a list. 55 * 56 * See sg_next(). 57 * 58 */ 59 60 #define SG_MAGIC 0x87654321 61 62 /* 63 * We overload the LSB of the page pointer to indicate whether it's 64 * a valid sg entry, or whether it points to the start of a new scatterlist. 65 * Those low bits are there for everyone! (thanks mason :-) 66 */ 67 #define sg_is_chain(sg) ((sg)->page_link & 0x01) 68 #define sg_is_last(sg) ((sg)->page_link & 0x02) 69 #define sg_chain_ptr(sg) \ 70 ((struct scatterlist *) ((sg)->page_link & ~0x03)) 71 72 /** 73 * sg_assign_page - Assign a given page to an SG entry 74 * @sg: SG entry 75 * @page: The page 76 * 77 * Description: 78 * Assign page to sg entry. Also see sg_set_page(), the most commonly used 79 * variant. 80 * 81 **/ 82 static inline void sg_assign_page(struct scatterlist *sg, struct page *page) 83 { 84 unsigned long page_link = sg->page_link & 0x3; 85 86 /* 87 * In order for the low bit stealing approach to work, pages 88 * must be aligned at a 32-bit boundary as a minimum. 89 */ 90 BUG_ON((unsigned long) page & 0x03); 91 #ifdef CONFIG_DEBUG_SG 92 BUG_ON(sg->sg_magic != SG_MAGIC); 93 BUG_ON(sg_is_chain(sg)); 94 #endif 95 sg->page_link = page_link | (unsigned long) page; 96 } 97 98 /** 99 * sg_set_page - Set sg entry to point at given page 100 * @sg: SG entry 101 * @page: The page 102 * @len: Length of data 103 * @offset: Offset into page 104 * 105 * Description: 106 * Use this function to set an sg entry pointing at a page, never assign 107 * the page directly. We encode sg table information in the lower bits 108 * of the page pointer. See sg_page() for looking up the page belonging 109 * to an sg entry. 110 * 111 **/ 112 static inline void sg_set_page(struct scatterlist *sg, struct page *page, 113 unsigned int len, unsigned int offset) 114 { 115 sg_assign_page(sg, page); 116 sg->offset = offset; 117 sg->length = len; 118 } 119 120 static inline struct page *sg_page(struct scatterlist *sg) 121 { 122 #ifdef CONFIG_DEBUG_SG 123 BUG_ON(sg->sg_magic != SG_MAGIC); 124 BUG_ON(sg_is_chain(sg)); 125 #endif 126 return (struct page *)((sg)->page_link & ~0x3); 127 } 128 129 /** 130 * sg_set_buf - Set sg entry to point at given data 131 * @sg: SG entry 132 * @buf: Data 133 * @buflen: Data length 134 * 135 **/ 136 static inline void sg_set_buf(struct scatterlist *sg, const void *buf, 137 unsigned int buflen) 138 { 139 #ifdef CONFIG_DEBUG_SG 140 BUG_ON(!virt_addr_valid(buf)); 141 #endif 142 sg_set_page(sg, virt_to_page(buf), buflen, offset_in_page(buf)); 143 } 144 145 /* 146 * Loop over each sg element, following the pointer to a new list if necessary 147 */ 148 #define for_each_sg(sglist, sg, nr, __i) \ 149 for (__i = 0, sg = (sglist); __i < (nr); __i++, sg = sg_next(sg)) 150 151 /** 152 * sg_chain - Chain two sglists together 153 * @prv: First scatterlist 154 * @prv_nents: Number of entries in prv 155 * @sgl: Second scatterlist 156 * 157 * Description: 158 * Links @prv@ and @sgl@ together, to form a longer scatterlist. 159 * 160 **/ 161 static inline void sg_chain(struct scatterlist *prv, unsigned int prv_nents, 162 struct scatterlist *sgl) 163 { 164 #ifndef CONFIG_ARCH_HAS_SG_CHAIN 165 BUG(); 166 #endif 167 168 /* 169 * offset and length are unused for chain entry. Clear them. 170 */ 171 prv[prv_nents - 1].offset = 0; 172 prv[prv_nents - 1].length = 0; 173 174 /* 175 * Set lowest bit to indicate a link pointer, and make sure to clear 176 * the termination bit if it happens to be set. 177 */ 178 prv[prv_nents - 1].page_link = ((unsigned long) sgl | 0x01) & ~0x02; 179 } 180 181 /** 182 * sg_mark_end - Mark the end of the scatterlist 183 * @sg: SG entryScatterlist 184 * 185 * Description: 186 * Marks the passed in sg entry as the termination point for the sg 187 * table. A call to sg_next() on this entry will return NULL. 188 * 189 **/ 190 static inline void sg_mark_end(struct scatterlist *sg) 191 { 192 #ifdef CONFIG_DEBUG_SG 193 BUG_ON(sg->sg_magic != SG_MAGIC); 194 #endif 195 /* 196 * Set termination bit, clear potential chain bit 197 */ 198 sg->page_link |= 0x02; 199 sg->page_link &= ~0x01; 200 } 201 202 /** 203 * sg_unmark_end - Undo setting the end of the scatterlist 204 * @sg: SG entryScatterlist 205 * 206 * Description: 207 * Removes the termination marker from the given entry of the scatterlist. 208 * 209 **/ 210 static inline void sg_unmark_end(struct scatterlist *sg) 211 { 212 #ifdef CONFIG_DEBUG_SG 213 BUG_ON(sg->sg_magic != SG_MAGIC); 214 #endif 215 sg->page_link &= ~0x02; 216 } 217 218 /** 219 * sg_phys - Return physical address of an sg entry 220 * @sg: SG entry 221 * 222 * Description: 223 * This calls page_to_phys() on the page in this sg entry, and adds the 224 * sg offset. The caller must know that it is legal to call page_to_phys() 225 * on the sg page. 226 * 227 **/ 228 static inline dma_addr_t sg_phys(struct scatterlist *sg) 229 { 230 return page_to_phys(sg_page(sg)) + sg->offset; 231 } 232 233 /** 234 * sg_virt - Return virtual address of an sg entry 235 * @sg: SG entry 236 * 237 * Description: 238 * This calls page_address() on the page in this sg entry, and adds the 239 * sg offset. The caller must know that the sg page has a valid virtual 240 * mapping. 241 * 242 **/ 243 static inline void *sg_virt(struct scatterlist *sg) 244 { 245 return page_address(sg_page(sg)) + sg->offset; 246 } 247 248 int sg_nents(struct scatterlist *sg); 249 int sg_nents_for_len(struct scatterlist *sg, u64 len); 250 struct scatterlist *sg_next(struct scatterlist *); 251 struct scatterlist *sg_last(struct scatterlist *s, unsigned int); 252 void sg_init_table(struct scatterlist *, unsigned int); 253 void sg_init_one(struct scatterlist *, const void *, unsigned int); 254 255 typedef struct scatterlist *(sg_alloc_fn)(unsigned int, gfp_t); 256 typedef void (sg_free_fn)(struct scatterlist *, unsigned int); 257 258 void __sg_free_table(struct sg_table *, unsigned int, bool, sg_free_fn *); 259 void sg_free_table(struct sg_table *); 260 int __sg_alloc_table(struct sg_table *, unsigned int, unsigned int, 261 struct scatterlist *, gfp_t, sg_alloc_fn *); 262 int sg_alloc_table(struct sg_table *, unsigned int, gfp_t); 263 int sg_alloc_table_from_pages(struct sg_table *sgt, 264 struct page **pages, unsigned int n_pages, 265 unsigned long offset, unsigned long size, 266 gfp_t gfp_mask); 267 268 size_t sg_copy_buffer(struct scatterlist *sgl, unsigned int nents, void *buf, 269 size_t buflen, off_t skip, bool to_buffer); 270 271 size_t sg_copy_from_buffer(struct scatterlist *sgl, unsigned int nents, 272 const void *buf, size_t buflen); 273 size_t sg_copy_to_buffer(struct scatterlist *sgl, unsigned int nents, 274 void *buf, size_t buflen); 275 276 size_t sg_pcopy_from_buffer(struct scatterlist *sgl, unsigned int nents, 277 const void *buf, size_t buflen, off_t skip); 278 size_t sg_pcopy_to_buffer(struct scatterlist *sgl, unsigned int nents, 279 void *buf, size_t buflen, off_t skip); 280 281 /* 282 * Maximum number of entries that will be allocated in one piece, if 283 * a list larger than this is required then chaining will be utilized. 284 */ 285 #define SG_MAX_SINGLE_ALLOC (PAGE_SIZE / sizeof(struct scatterlist)) 286 287 /* 288 * sg page iterator 289 * 290 * Iterates over sg entries page-by-page. On each successful iteration, 291 * you can call sg_page_iter_page(@piter) and sg_page_iter_dma_address(@piter) 292 * to get the current page and its dma address. @piter->sg will point to the 293 * sg holding this page and @piter->sg_pgoffset to the page's page offset 294 * within the sg. The iteration will stop either when a maximum number of sg 295 * entries was reached or a terminating sg (sg_last(sg) == true) was reached. 296 */ 297 struct sg_page_iter { 298 struct scatterlist *sg; /* sg holding the page */ 299 unsigned int sg_pgoffset; /* page offset within the sg */ 300 301 /* these are internal states, keep away */ 302 unsigned int __nents; /* remaining sg entries */ 303 int __pg_advance; /* nr pages to advance at the 304 * next step */ 305 }; 306 307 bool __sg_page_iter_next(struct sg_page_iter *piter); 308 void __sg_page_iter_start(struct sg_page_iter *piter, 309 struct scatterlist *sglist, unsigned int nents, 310 unsigned long pgoffset); 311 /** 312 * sg_page_iter_page - get the current page held by the page iterator 313 * @piter: page iterator holding the page 314 */ 315 static inline struct page *sg_page_iter_page(struct sg_page_iter *piter) 316 { 317 return nth_page(sg_page(piter->sg), piter->sg_pgoffset); 318 } 319 320 /** 321 * sg_page_iter_dma_address - get the dma address of the current page held by 322 * the page iterator. 323 * @piter: page iterator holding the page 324 */ 325 static inline dma_addr_t sg_page_iter_dma_address(struct sg_page_iter *piter) 326 { 327 return sg_dma_address(piter->sg) + (piter->sg_pgoffset << PAGE_SHIFT); 328 } 329 330 /** 331 * for_each_sg_page - iterate over the pages of the given sg list 332 * @sglist: sglist to iterate over 333 * @piter: page iterator to hold current page, sg, sg_pgoffset 334 * @nents: maximum number of sg entries to iterate over 335 * @pgoffset: starting page offset 336 */ 337 #define for_each_sg_page(sglist, piter, nents, pgoffset) \ 338 for (__sg_page_iter_start((piter), (sglist), (nents), (pgoffset)); \ 339 __sg_page_iter_next(piter);) 340 341 /* 342 * Mapping sg iterator 343 * 344 * Iterates over sg entries mapping page-by-page. On each successful 345 * iteration, @miter->page points to the mapped page and 346 * @miter->length bytes of data can be accessed at @miter->addr. As 347 * long as an interation is enclosed between start and stop, the user 348 * is free to choose control structure and when to stop. 349 * 350 * @miter->consumed is set to @miter->length on each iteration. It 351 * can be adjusted if the user can't consume all the bytes in one go. 352 * Also, a stopped iteration can be resumed by calling next on it. 353 * This is useful when iteration needs to release all resources and 354 * continue later (e.g. at the next interrupt). 355 */ 356 357 #define SG_MITER_ATOMIC (1 << 0) /* use kmap_atomic */ 358 #define SG_MITER_TO_SG (1 << 1) /* flush back to phys on unmap */ 359 #define SG_MITER_FROM_SG (1 << 2) /* nop */ 360 361 struct sg_mapping_iter { 362 /* the following three fields can be accessed directly */ 363 struct page *page; /* currently mapped page */ 364 void *addr; /* pointer to the mapped area */ 365 size_t length; /* length of the mapped area */ 366 size_t consumed; /* number of consumed bytes */ 367 struct sg_page_iter piter; /* page iterator */ 368 369 /* these are internal states, keep away */ 370 unsigned int __offset; /* offset within page */ 371 unsigned int __remaining; /* remaining bytes on page */ 372 unsigned int __flags; 373 }; 374 375 void sg_miter_start(struct sg_mapping_iter *miter, struct scatterlist *sgl, 376 unsigned int nents, unsigned int flags); 377 bool sg_miter_skip(struct sg_mapping_iter *miter, off_t offset); 378 bool sg_miter_next(struct sg_mapping_iter *miter); 379 void sg_miter_stop(struct sg_mapping_iter *miter); 380 381 #endif /* _LINUX_SCATTERLIST_H */ 382