65 	struct scatterlist *sg;  member
86 #define	sg_is_chain(sg)		((sg)->page_link & SG_PAGE_LINK_CHAIN)  argument
87 #define	sg_is_last(sg)		((sg)->page_link & SG_PAGE_LINK_LAST)  argument
88 #define	sg_chain_ptr(sg)	\  argument
89 	((struct scatterlist *) ((sg)->page_link & ~SG_PAGE_LINK_MASK))
91 #define	sg_dma_address(sg)	(sg)->dma_address  argument
92 #define	sg_dma_len(sg)		(sg)->length  argument
96 	     (iter)->sg; _sg_iter_next(iter))
100 #define	for_each_sg(sglist, sg, sgmax, iter)				\  argument
101 	for (iter = 0, sg = (sglist); iter < (sgmax); iter++, sg = sg_next(sg))
103 #define	for_each_sgtable_sg(sgt, sg, i) \  argument
104 	for_each_sg((sgt)->sgl, sg, (sgt)->orig_nents, i)
109 #define	for_each_sgtable_dma_sg(sgt, sg, iter)				\  argument
110 	for_each_sg((sgt)->sgl, sg, (sgt)->nents, iter)
119 sg_assign_page(struct scatterlist *sg, struct page *page)  in sg_assign_page()  argument
121 	unsigned long page_link = sg->page_link & SG_PAGE_LINK_MASK;  in sg_assign_page()
123 	sg->page_link = page_link | (unsigned long)page;  in sg_assign_page()
127 sg_set_page(struct scatterlist *sg, struct page *page, unsigned int len,  in sg_set_page()  argument
130 	sg_assign_page(sg, page);  in sg_set_page()
131 	sg->offset = offset;  in sg_set_page()
132 	sg->length = len;  in sg_set_page()
136 sg_page(struct scatterlist *sg)  in sg_page()  argument
138 	return ((struct page *)((sg)->page_link & ~SG_PAGE_LINK_MASK));  in sg_page()
142 sg_set_buf(struct scatterlist *sg, const void *buf, unsigned int buflen)  in sg_set_buf()  argument
144 	sg_set_page(sg, virt_to_page(buf), buflen,  in sg_set_buf()
149 sg_next(struct scatterlist *sg)  in sg_next()  argument
151 	if (sg_is_last(sg))  in sg_next()
153 	sg++;  in sg_next()
154 	if (sg_is_chain(sg))  in sg_next()
155 		sg = sg_chain_ptr(sg);  in sg_next()
156 	return (sg);  in sg_next()
160 sg_phys(struct scatterlist *sg)  in sg_phys()  argument
162 	return (page_to_phys(sg_page(sg)) + sg->offset);  in sg_phys()
166 sg_virt(struct scatterlist *sg)  in sg_virt()  argument
169 	return ((void *)((unsigned long)page_address(sg_page(sg)) + sg->offset));  in sg_virt()
176 	struct scatterlist *sg = &prv[prv_nents - 1];  in sg_chain()  local
178 	sg->offset = 0;  in sg_chain()
179 	sg->length = 0;  in sg_chain()
180 	sg->page_link = ((unsigned long)sgl |  in sg_chain()
185 sg_mark_end(struct scatterlist *sg)  in sg_mark_end()  argument
187 	sg->page_link |= SG_PAGE_LINK_LAST;  in sg_mark_end()
188 	sg->page_link &= ~SG_PAGE_LINK_CHAIN;  in sg_mark_end()
192 sg_init_table(struct scatterlist *sg, unsigned int nents)  in sg_init_table()  argument
194 	bzero(sg, sizeof(*sg) * nents);  in sg_init_table()
195 	sg_mark_end(&sg[nents - 1]);  in sg_init_table()
199 sg_init_one(struct scatterlist *sg, const void *buf, unsigned int buflen)  in sg_init_one()  argument
201 	sg_init_table(sg, 1);  in sg_init_one()
202 	sg_set_buf(sg, buf, buflen);  in sg_init_one()
215 sg_kfree(struct scatterlist *sg, unsigned int nents)  in sg_kfree()  argument
218 		free_page((unsigned long)sg);  in sg_kfree()
220 		kfree(sg);  in sg_kfree()
268 	struct scatterlist *sg, *prv;  in __sg_alloc_table()  local
290 			sg = first_chunk;  in __sg_alloc_table()
293 			sg = alloc_fn(alloc_size, gfp_mask);  in __sg_alloc_table()
295 		if (unlikely(!sg)) {  in __sg_alloc_table()
301 		sg_init_table(sg, alloc_size);  in __sg_alloc_table()
305 			sg_chain(prv, max_ents, sg);  in __sg_alloc_table()
307 			table->sgl = sg;  in __sg_alloc_table()
310 			sg_mark_end(&sg[sg_size - 1]);  in __sg_alloc_table()
312 		prv = sg;  in __sg_alloc_table()
452 sg_nents(struct scatterlist *sg)  in sg_nents()  argument
456 	for (nents = 0; sg; sg = sg_next(sg))  in sg_nents()
469 	piter->sg = sglist;  in __sg_page_iter_start()
476 	struct scatterlist *sg;  in _sg_iter_next()  local
479 	sg = iter->sg;  in _sg_iter_next()
480 	pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT;  in _sg_iter_next()
485 		sg = sg_next(sg);  in _sg_iter_next()
487 		if (sg == NULL || iter->maxents == 0)  in _sg_iter_next()
489 		pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT;  in _sg_iter_next()
491 	iter->sg = sg;  in _sg_iter_next()
495 sg_page_count(struct scatterlist *sg)  in sg_page_count()  argument
497 	return (PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT);  in sg_page_count()
499 #define	sg_dma_page_count(sg) \  argument
500 	sg_page_count(sg)
509 	if (piter->sg == NULL)  in __sg_page_iter_next()
516 		pgcount = sg_page_count(piter->sg);  in __sg_page_iter_next()
520 		piter->sg = sg_next(piter->sg);  in __sg_page_iter_next()
523 		if (piter->sg == NULL)  in __sg_page_iter_next()
536 		iter->sg = sgl;  in _sg_iter_init()
541 		iter->sg = NULL;  in _sg_iter_init()
557 	__dma_address = __spi->sg->dma_address +	\
565 	return (nth_page(sg_page(piter->sg), piter->sg_pgoffset));  in sg_page_iter_page()
587 		if (piter.sg->length <= skip) {  in sg_pcopy_from_buffer()
588 			skip -= piter.sg->length;  in sg_pcopy_from_buffer()
593 		KASSERT(((piter.sg->length - skip) != 0 && (buflen != 0)),  in sg_pcopy_from_buffer()
595 		    __func__, piter.sg->length, (uintmax_t)skip, buflen));  in sg_pcopy_from_buffer()
596 		len = min(piter.sg->length - skip, buflen);  in sg_pcopy_from_buffer()
632 	struct scatterlist *sg;  in sg_pcopy_to_buffer()  local
642 		sg = iter.sg;  in sg_pcopy_to_buffer()
644 		if (offset >= sg->length) {  in sg_pcopy_to_buffer()
645 			offset -= sg->length;  in sg_pcopy_to_buffer()
648 		len = ulmin(buflen, sg->length - offset);  in sg_pcopy_to_buffer()
660 		memcpy(buf, vaddr + sg->offset + offset, len);  in sg_pcopy_to_buffer()