22 	void *(*alloc)(struct snd_dma_buffer *dmab, size_t size);
23 	void (*free)(struct snd_dma_buffer *dmab);
24 	dma_addr_t (*get_addr)(struct snd_dma_buffer *dmab, size_t offset);
25 	struct page *(*get_page)(struct snd_dma_buffer *dmab, size_t offset);
26 	unsigned int (*get_chunk_size)(struct snd_dma_buffer *dmab,
28 	int (*mmap)(struct snd_dma_buffer *dmab, struct vm_area_struct *area);
29 	void (*sync)(struct snd_dma_buffer *dmab, enum snd_dma_sync_mode mode);
37 static const struct snd_malloc_ops *snd_dma_get_ops(struct snd_dma_buffer *dmab);
39 static void *__snd_dma_alloc_pages(struct snd_dma_buffer *dmab, size_t size)  in __snd_dma_alloc_pages()  argument
41 	const struct snd_malloc_ops *ops = snd_dma_get_ops(dmab);  in __snd_dma_alloc_pages()
45 	return ops->alloc(dmab, size);  in __snd_dma_alloc_pages()
65 			    struct snd_dma_buffer *dmab)  in snd_dma_alloc_dir_pages()  argument
69 	if (WARN_ON(!dmab))  in snd_dma_alloc_dir_pages()
73 	dmab->dev.type = type;  in snd_dma_alloc_dir_pages()
74 	dmab->dev.dev = device;  in snd_dma_alloc_dir_pages()
75 	dmab->dev.dir = dir;  in snd_dma_alloc_dir_pages()
76 	dmab->bytes = 0;  in snd_dma_alloc_dir_pages()
77 	dmab->addr = 0;  in snd_dma_alloc_dir_pages()
78 	dmab->private_data = NULL;  in snd_dma_alloc_dir_pages()
79 	dmab->area = __snd_dma_alloc_pages(dmab, size);  in snd_dma_alloc_dir_pages()
80 	if (!dmab->area)  in snd_dma_alloc_dir_pages()
82 	dmab->bytes = size;  in snd_dma_alloc_dir_pages()
103 				 struct snd_dma_buffer *dmab)  in snd_dma_alloc_pages_fallback()  argument
107 	while ((err = snd_dma_alloc_pages(type, device, size, dmab)) < 0) {  in snd_dma_alloc_pages_fallback()
115 	if (! dmab->area)  in snd_dma_alloc_pages_fallback()
127 void snd_dma_free_pages(struct snd_dma_buffer *dmab)  in snd_dma_free_pages()  argument
129 	const struct snd_malloc_ops *ops = snd_dma_get_ops(dmab);  in snd_dma_free_pages()
132 		ops->free(dmab);  in snd_dma_free_pages()
162 	struct snd_dma_buffer *dmab;  in snd_devm_alloc_dir_pages()  local
169 	dmab = devres_alloc(__snd_release_pages, sizeof(*dmab), GFP_KERNEL);  in snd_devm_alloc_dir_pages()
170 	if (!dmab)  in snd_devm_alloc_dir_pages()
173 	err = snd_dma_alloc_dir_pages(type, dev, dir, size, dmab);  in snd_devm_alloc_dir_pages()
175 		devres_free(dmab);  in snd_devm_alloc_dir_pages()
179 	devres_add(dev, dmab);  in snd_devm_alloc_dir_pages()
180 	return dmab;  in snd_devm_alloc_dir_pages()
191 int snd_dma_buffer_mmap(struct snd_dma_buffer *dmab,  in snd_dma_buffer_mmap()  argument
196 	if (!dmab)  in snd_dma_buffer_mmap()
198 	ops = snd_dma_get_ops(dmab);  in snd_dma_buffer_mmap()
200 		return ops->mmap(dmab, area);  in snd_dma_buffer_mmap()
212 void snd_dma_buffer_sync(struct snd_dma_buffer *dmab,  in snd_dma_buffer_sync()  argument
217 	if (!dmab || !dmab->dev.need_sync)  in snd_dma_buffer_sync()
219 	ops = snd_dma_get_ops(dmab);  in snd_dma_buffer_sync()
221 		ops->sync(dmab, mode);  in snd_dma_buffer_sync()
233 dma_addr_t snd_sgbuf_get_addr(struct snd_dma_buffer *dmab, size_t offset)  in snd_sgbuf_get_addr()  argument
235 	const struct snd_malloc_ops *ops = snd_dma_get_ops(dmab);  in snd_sgbuf_get_addr()
238 		return ops->get_addr(dmab, offset);  in snd_sgbuf_get_addr()
240 		return dmab->addr + offset;  in snd_sgbuf_get_addr()
251 struct page *snd_sgbuf_get_page(struct snd_dma_buffer *dmab, size_t offset)  in snd_sgbuf_get_page()  argument
253 	const struct snd_malloc_ops *ops = snd_dma_get_ops(dmab);  in snd_sgbuf_get_page()
256 		return ops->get_page(dmab, offset);  in snd_sgbuf_get_page()
258 		return virt_to_page(dmab->area + offset);  in snd_sgbuf_get_page()
271 unsigned int snd_sgbuf_get_chunk_size(struct snd_dma_buffer *dmab,  in snd_sgbuf_get_chunk_size()  argument
274 	const struct snd_malloc_ops *ops = snd_dma_get_ops(dmab);  in snd_sgbuf_get_chunk_size()
277 		return ops->get_chunk_size(dmab, ofs, size);  in snd_sgbuf_get_chunk_size()
326 static void *snd_dma_continuous_alloc(struct snd_dma_buffer *dmab, size_t size)  in snd_dma_continuous_alloc()  argument
328 	return do_alloc_pages(dmab->dev.dev, size, &dmab->addr, false);  in snd_dma_continuous_alloc()
331 static void snd_dma_continuous_free(struct snd_dma_buffer *dmab)  in snd_dma_continuous_free()  argument
333 	do_free_pages(dmab->area, dmab->bytes, false);  in snd_dma_continuous_free()
336 static int snd_dma_continuous_mmap(struct snd_dma_buffer *dmab,  in snd_dma_continuous_mmap()  argument
340 			       dmab->addr >> PAGE_SHIFT,  in snd_dma_continuous_mmap()
354 static void *snd_dma_vmalloc_alloc(struct snd_dma_buffer *dmab, size_t size)  in snd_dma_vmalloc_alloc()  argument
359 static void snd_dma_vmalloc_free(struct snd_dma_buffer *dmab)  in snd_dma_vmalloc_free()  argument
361 	vfree(dmab->area);  in snd_dma_vmalloc_free()
364 static int snd_dma_vmalloc_mmap(struct snd_dma_buffer *dmab,  in snd_dma_vmalloc_mmap()  argument
367 	return remap_vmalloc_range(area, dmab->area, 0);  in snd_dma_vmalloc_mmap()
370 #define get_vmalloc_page_addr(dmab, offset) \  argument
371 	page_to_phys(vmalloc_to_page((dmab)->area + (offset)))
373 static dma_addr_t snd_dma_vmalloc_get_addr(struct snd_dma_buffer *dmab,  in snd_dma_vmalloc_get_addr()  argument
376 	return get_vmalloc_page_addr(dmab, offset) + offset % PAGE_SIZE;  in snd_dma_vmalloc_get_addr()
379 static struct page *snd_dma_vmalloc_get_page(struct snd_dma_buffer *dmab,  in snd_dma_vmalloc_get_page()  argument
382 	return vmalloc_to_page(dmab->area + offset);  in snd_dma_vmalloc_get_page()
386 snd_dma_vmalloc_get_chunk_size(struct snd_dma_buffer *dmab,  in snd_dma_vmalloc_get_chunk_size()  argument
395 	addr = get_vmalloc_page_addr(dmab, start);  in snd_dma_vmalloc_get_chunk_size()
401 		if (get_vmalloc_page_addr(dmab, start) != addr)  in snd_dma_vmalloc_get_chunk_size()
422 static void *snd_dma_iram_alloc(struct snd_dma_buffer *dmab, size_t size)  in snd_dma_iram_alloc()  argument
424 	struct device *dev = dmab->dev.dev;  in snd_dma_iram_alloc()
431 		dmab->private_data = pool;  in snd_dma_iram_alloc()
433 		p = gen_pool_dma_alloc_align(pool, size, &dmab->addr, PAGE_SIZE);  in snd_dma_iram_alloc()
441 	dmab->dev.type = SNDRV_DMA_TYPE_DEV;  in snd_dma_iram_alloc()
442 	return __snd_dma_alloc_pages(dmab, size);  in snd_dma_iram_alloc()
445 static void snd_dma_iram_free(struct snd_dma_buffer *dmab)  in snd_dma_iram_free()  argument
447 	struct gen_pool *pool = dmab->private_data;  in snd_dma_iram_free()
449 	if (pool && dmab->area)  in snd_dma_iram_free()
450 		gen_pool_free(pool, (unsigned long)dmab->area, dmab->bytes);  in snd_dma_iram_free()
453 static int snd_dma_iram_mmap(struct snd_dma_buffer *dmab,  in snd_dma_iram_mmap()  argument
458 			       dmab->addr >> PAGE_SHIFT,  in snd_dma_iram_mmap()
473 static void *snd_dma_dev_alloc(struct snd_dma_buffer *dmab, size_t size)  in snd_dma_dev_alloc()  argument
475 	return dma_alloc_coherent(dmab->dev.dev, size, &dmab->addr, DEFAULT_GFP);  in snd_dma_dev_alloc()
478 static void snd_dma_dev_free(struct snd_dma_buffer *dmab)  in snd_dma_dev_free()  argument
480 	dma_free_coherent(dmab->dev.dev, dmab->bytes, dmab->area, dmab->addr);  in snd_dma_dev_free()
483 static int snd_dma_dev_mmap(struct snd_dma_buffer *dmab,  in snd_dma_dev_mmap()  argument
486 	return dma_mmap_coherent(dmab->dev.dev, area,  in snd_dma_dev_mmap()
487 				 dmab->area, dmab->addr, dmab->bytes);  in snd_dma_dev_mmap()
501 static void *snd_dma_wc_alloc(struct snd_dma_buffer *dmab, size_t size)  in snd_dma_wc_alloc()  argument
503 	void *p = do_alloc_pages(dmab->dev.dev, size, &dmab->addr, true);  in snd_dma_wc_alloc()
507 	dmab->addr = dma_map_single(dmab->dev.dev, p, size, DMA_BIDIRECTIONAL);  in snd_dma_wc_alloc()
508 	if (dma_mapping_error(dmab->dev.dev, dmab->addr)) {  in snd_dma_wc_alloc()
509 		do_free_pages(dmab->area, size, true);  in snd_dma_wc_alloc()
515 static void snd_dma_wc_free(struct snd_dma_buffer *dmab)  in snd_dma_wc_free()  argument
517 	dma_unmap_single(dmab->dev.dev, dmab->addr, dmab->bytes,  in snd_dma_wc_free()
519 	do_free_pages(dmab->area, dmab->bytes, true);  in snd_dma_wc_free()
522 static int snd_dma_wc_mmap(struct snd_dma_buffer *dmab,  in snd_dma_wc_mmap()  argument
526 	return dma_mmap_coherent(dmab->dev.dev, area,  in snd_dma_wc_mmap()
527 				 dmab->area, dmab->addr, dmab->bytes);  in snd_dma_wc_mmap()
530 static void *snd_dma_wc_alloc(struct snd_dma_buffer *dmab, size_t size)  in snd_dma_wc_alloc()  argument
532 	return dma_alloc_wc(dmab->dev.dev, size, &dmab->addr, DEFAULT_GFP);  in snd_dma_wc_alloc()
535 static void snd_dma_wc_free(struct snd_dma_buffer *dmab)  in snd_dma_wc_free()  argument
537 	dma_free_wc(dmab->dev.dev, dmab->bytes, dmab->area, dmab->addr);  in snd_dma_wc_free()
540 static int snd_dma_wc_mmap(struct snd_dma_buffer *dmab,  in snd_dma_wc_mmap()  argument
543 	return dma_mmap_wc(dmab->dev.dev, area,  in snd_dma_wc_mmap()
544 			   dmab->area, dmab->addr, dmab->bytes);  in snd_dma_wc_mmap()
557 static void *snd_dma_noncontig_alloc(struct snd_dma_buffer *dmab, size_t size)  in snd_dma_noncontig_alloc()  argument
562 	sgt = dma_alloc_noncontiguous(dmab->dev.dev, size, dmab->dev.dir,  in snd_dma_noncontig_alloc()
567 	dmab->dev.need_sync = dma_need_sync(dmab->dev.dev,  in snd_dma_noncontig_alloc()
569 	p = dma_vmap_noncontiguous(dmab->dev.dev, size, sgt);  in snd_dma_noncontig_alloc()
571 		dmab->private_data = sgt;  in snd_dma_noncontig_alloc()
573 		dmab->addr = snd_sgbuf_get_addr(dmab, 0);  in snd_dma_noncontig_alloc()
575 		dma_free_noncontiguous(dmab->dev.dev, size, sgt, dmab->dev.dir);  in snd_dma_noncontig_alloc()
580 static void snd_dma_noncontig_free(struct snd_dma_buffer *dmab)  in snd_dma_noncontig_free()  argument
582 	dma_vunmap_noncontiguous(dmab->dev.dev, dmab->area);  in snd_dma_noncontig_free()
583 	dma_free_noncontiguous(dmab->dev.dev, dmab->bytes, dmab->private_data,  in snd_dma_noncontig_free()
584 			       dmab->dev.dir);  in snd_dma_noncontig_free()
587 static int snd_dma_noncontig_mmap(struct snd_dma_buffer *dmab,  in snd_dma_noncontig_mmap()  argument
590 	return dma_mmap_noncontiguous(dmab->dev.dev, area,  in snd_dma_noncontig_mmap()
591 				      dmab->bytes, dmab->private_data);  in snd_dma_noncontig_mmap()
594 static void snd_dma_noncontig_sync(struct snd_dma_buffer *dmab,  in snd_dma_noncontig_sync()  argument
598 		if (dmab->dev.dir == DMA_TO_DEVICE)  in snd_dma_noncontig_sync()
600 		invalidate_kernel_vmap_range(dmab->area, dmab->bytes);  in snd_dma_noncontig_sync()
601 		dma_sync_sgtable_for_cpu(dmab->dev.dev, dmab->private_data,  in snd_dma_noncontig_sync()
602 					 dmab->dev.dir);  in snd_dma_noncontig_sync()
604 		if (dmab->dev.dir == DMA_FROM_DEVICE)  in snd_dma_noncontig_sync()
606 		flush_kernel_vmap_range(dmab->area, dmab->bytes);  in snd_dma_noncontig_sync()
607 		dma_sync_sgtable_for_device(dmab->dev.dev, dmab->private_data,  in snd_dma_noncontig_sync()
608 					    dmab->dev.dir);  in snd_dma_noncontig_sync()
612 static inline void snd_dma_noncontig_iter_set(struct snd_dma_buffer *dmab,  in snd_dma_noncontig_iter_set()  argument
616 	struct sg_table *sgt = dmab->private_data;  in snd_dma_noncontig_iter_set()
622 static dma_addr_t snd_dma_noncontig_get_addr(struct snd_dma_buffer *dmab,  in snd_dma_noncontig_get_addr()  argument
627 	snd_dma_noncontig_iter_set(dmab, &iter.base, offset);  in snd_dma_noncontig_get_addr()
632 static struct page *snd_dma_noncontig_get_page(struct snd_dma_buffer *dmab,  in snd_dma_noncontig_get_page()  argument
637 	snd_dma_noncontig_iter_set(dmab, &iter, offset);  in snd_dma_noncontig_get_page()
643 snd_dma_noncontig_get_chunk_size(struct snd_dma_buffer *dmab,  in snd_dma_noncontig_get_chunk_size()  argument
652 	snd_dma_noncontig_iter_set(dmab, &iter.base, start);  in snd_dma_noncontig_get_chunk_size()
689 static void __snd_dma_sg_fallback_free(struct snd_dma_buffer *dmab,  in __snd_dma_sg_fallback_free()  argument
692 	bool wc = dmab->dev.type == SNDRV_DMA_TYPE_DEV_WC_SG;  in __snd_dma_sg_fallback_free()
712 static void *snd_dma_sg_fallback_alloc(struct snd_dma_buffer *dmab, size_t size)  in snd_dma_sg_fallback_alloc()  argument
714 	bool wc = dmab->dev.type == SNDRV_DMA_TYPE_DEV_WC_SG;  in snd_dma_sg_fallback_alloc()
737 		p = do_alloc_pages(dmab->dev.dev, chunk, &addr, wc);  in snd_dma_sg_fallback_alloc()
760 	if (dma_map_sgtable(dmab->dev.dev, &sgbuf->sgt, DMA_BIDIRECTIONAL, 0))  in snd_dma_sg_fallback_alloc()
767 	dmab->private_data = sgbuf;  in snd_dma_sg_fallback_alloc()
769 	dmab->addr = snd_sgbuf_get_addr(dmab, 0);  in snd_dma_sg_fallback_alloc()
773 	dma_unmap_sgtable(dmab->dev.dev, &sgbuf->sgt, DMA_BIDIRECTIONAL, 0);  in snd_dma_sg_fallback_alloc()
777 	__snd_dma_sg_fallback_free(dmab, sgbuf);  in snd_dma_sg_fallback_alloc()
781 static void snd_dma_sg_fallback_free(struct snd_dma_buffer *dmab)  in snd_dma_sg_fallback_free()  argument
783 	struct snd_dma_sg_fallback *sgbuf = dmab->private_data;  in snd_dma_sg_fallback_free()
785 	vunmap(dmab->area);  in snd_dma_sg_fallback_free()
786 	dma_unmap_sgtable(dmab->dev.dev, &sgbuf->sgt, DMA_BIDIRECTIONAL, 0);  in snd_dma_sg_fallback_free()
788 	__snd_dma_sg_fallback_free(dmab, dmab->private_data);  in snd_dma_sg_fallback_free()
791 static int snd_dma_sg_fallback_mmap(struct snd_dma_buffer *dmab,  in snd_dma_sg_fallback_mmap()  argument
794 	struct snd_dma_sg_fallback *sgbuf = dmab->private_data;  in snd_dma_sg_fallback_mmap()
796 	if (dmab->dev.type == SNDRV_DMA_TYPE_DEV_WC_SG)  in snd_dma_sg_fallback_mmap()
801 static void *snd_dma_sg_alloc(struct snd_dma_buffer *dmab, size_t size)  in snd_dma_sg_alloc()  argument
803 	int type = dmab->dev.type;  in snd_dma_sg_alloc()
808 		dmab->dev.type = SNDRV_DMA_TYPE_DEV_WC;  in snd_dma_sg_alloc()
810 		dmab->dev.type = SNDRV_DMA_TYPE_DEV;  in snd_dma_sg_alloc()
811 	p = __snd_dma_alloc_pages(dmab, size);  in snd_dma_sg_alloc()
815 	dmab->dev.type = type; /* restore the type */  in snd_dma_sg_alloc()
816 	return snd_dma_sg_fallback_alloc(dmab, size);  in snd_dma_sg_alloc()
834 static void *snd_dma_noncoherent_alloc(struct snd_dma_buffer *dmab, size_t size)  in snd_dma_noncoherent_alloc()  argument
838 	p = dma_alloc_noncoherent(dmab->dev.dev, size, &dmab->addr,  in snd_dma_noncoherent_alloc()
839 				  dmab->dev.dir, DEFAULT_GFP);  in snd_dma_noncoherent_alloc()
841 		dmab->dev.need_sync = dma_need_sync(dmab->dev.dev, dmab->addr);  in snd_dma_noncoherent_alloc()
845 static void snd_dma_noncoherent_free(struct snd_dma_buffer *dmab)  in snd_dma_noncoherent_free()  argument
847 	dma_free_noncoherent(dmab->dev.dev, dmab->bytes, dmab->area,  in snd_dma_noncoherent_free()
848 			     dmab->addr, dmab->dev.dir);  in snd_dma_noncoherent_free()
851 static int snd_dma_noncoherent_mmap(struct snd_dma_buffer *dmab,  in snd_dma_noncoherent_mmap()  argument
855 	return dma_mmap_pages(dmab->dev.dev, area,  in snd_dma_noncoherent_mmap()
857 			      virt_to_page(dmab->area));  in snd_dma_noncoherent_mmap()
860 static void snd_dma_noncoherent_sync(struct snd_dma_buffer *dmab,  in snd_dma_noncoherent_sync()  argument
864 		if (dmab->dev.dir != DMA_TO_DEVICE)  in snd_dma_noncoherent_sync()
865 			dma_sync_single_for_cpu(dmab->dev.dev, dmab->addr,  in snd_dma_noncoherent_sync()
866 						dmab->bytes, dmab->dev.dir);  in snd_dma_noncoherent_sync()
868 		if (dmab->dev.dir != DMA_FROM_DEVICE)  in snd_dma_noncoherent_sync()
869 			dma_sync_single_for_device(dmab->dev.dev, dmab->addr,  in snd_dma_noncoherent_sync()
870 						   dmab->bytes, dmab->dev.dir);  in snd_dma_noncoherent_sync()
904 static const struct snd_malloc_ops *snd_dma_get_ops(struct snd_dma_buffer *dmab)  in snd_dma_get_ops()  argument
906 	if (WARN_ON_ONCE(!dmab))  in snd_dma_get_ops()
908 	if (WARN_ON_ONCE(dmab->dev.type <= SNDRV_DMA_TYPE_UNKNOWN ||  in snd_dma_get_ops()
909 			 dmab->dev.type >= ARRAY_SIZE(snd_dma_ops)))  in snd_dma_get_ops()
911 	return snd_dma_ops[dmab->dev.type];  in snd_dma_get_ops()