86 usbd_get_page(struct usb_page_cache *pc, usb_frlength_t offset,  in usbd_get_page()  argument
92 	if (pc->page_start) {  in usbd_get_page()
95 		if (pc->buffer) {  in usbd_get_page()
98 			res->buffer = USB_ADD_BYTES(pc->buffer, offset);  in usbd_get_page()
100 		offset += pc->page_offset_buf;  in usbd_get_page()
104 		page = pc->page_start;  in usbd_get_page()
106 		if (pc->ismultiseg) {  in usbd_get_page()
117 		if (!pc->buffer) {  in usbd_get_page()
127 	res->buffer = USB_ADD_BYTES(pc->buffer, offset);  in usbd_get_page()
141 usb_pc_buffer_is_aligned(struct usb_page_cache *pc, usb_frlength_t offset,  in usb_pc_buffer_is_aligned()  argument
147 		usbd_get_page(pc, offset, &buf_res);  in usb_pc_buffer_is_aligned()
251 usb_uiomove(struct usb_page_cache *pc, struct uio *uio,  in usb_uiomove()  argument
258 		usbd_get_page(pc, pc_offset, &res);  in usb_uiomove()
439 	struct usb_page_cache *pc;  in usb_pc_common_mem_cb()  local
445 	pc = arg;  in usb_pc_common_mem_cb()
446 	uptag = pc->tag_parent;  in usb_pc_common_mem_cb()
460 	pg = pc->page_start;  in usb_pc_common_mem_cb()
463 	pc->page_offset_buf = rem;  in usb_pc_common_mem_cb()
464 	pc->page_offset_end += rem;  in usb_pc_common_mem_cb()
484 	while (pc->ismultiseg) {  in usb_pc_common_mem_cb()
522 usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg,  in usb_pc_alloc_mem()  argument
531 	uptag = pc->tag_parent;  in usb_pc_alloc_mem()
578 	pc->buffer = ptr;  in usb_pc_alloc_mem()
579 	pc->page_start = pg;  in usb_pc_alloc_mem()
580 	pc->page_offset_buf = 0;  in usb_pc_alloc_mem()
581 	pc->page_offset_end = size;  in usb_pc_alloc_mem()
582 	pc->map = map;  in usb_pc_alloc_mem()
583 	pc->tag = utag->tag;  in usb_pc_alloc_mem()
584 	pc->ismultiseg = (align == 1);  in usb_pc_alloc_mem()
591 	    pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT));  in usb_pc_alloc_mem()
603 	pc->isloaded = 1;  in usb_pc_alloc_mem()
606 	usb_pc_cpu_flush(pc);  in usb_pc_alloc_mem()
612 	pc->buffer = NULL;  in usb_pc_alloc_mem()
613 	pc->page_start = NULL;  in usb_pc_alloc_mem()
614 	pc->page_offset_buf = 0;  in usb_pc_alloc_mem()
615 	pc->page_offset_end = 0;  in usb_pc_alloc_mem()
616 	pc->isloaded = 0;  in usb_pc_alloc_mem()
617 	pc->map = NULL;  in usb_pc_alloc_mem()
618 	pc->tag = NULL;  in usb_pc_alloc_mem()
628 usb_pc_free_mem(struct usb_page_cache *pc)  in usb_pc_free_mem()  argument
630 	if (pc && pc->buffer) {  in usb_pc_free_mem()
631 		if (pc->isloaded)  in usb_pc_free_mem()
632 			bus_dmamap_unload(pc->tag, pc->map);  in usb_pc_free_mem()
634 		bus_dmamem_free(pc->tag, pc->buffer, pc->map);  in usb_pc_free_mem()
636 		pc->buffer = NULL;  in usb_pc_free_mem()
637 		pc->isloaded = 0;  in usb_pc_free_mem()
649 usb_pc_load_mem(struct usb_page_cache *pc, usb_size_t size, uint8_t sync)  in usb_pc_load_mem()  argument
652 	pc->page_offset_buf = 0;  in usb_pc_load_mem()
653 	pc->page_offset_end = size;  in usb_pc_load_mem()
654 	pc->ismultiseg = 1;  in usb_pc_load_mem()
656 	USB_MTX_ASSERT(pc->tag_parent->mtx, MA_OWNED);  in usb_pc_load_mem()
663 			uptag = pc->tag_parent;  in usb_pc_load_mem()
669 			if (pc->isloaded)  in usb_pc_load_mem()
670 				bus_dmamap_unload(pc->tag, pc->map);  in usb_pc_load_mem()
676 			    pc->tag, pc->map, pc->buffer, size,  in usb_pc_load_mem()
677 			    &usb_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK);  in usb_pc_load_mem()
683 				pc->isloaded = 0;  in usb_pc_load_mem()
691 			if (pc->isloaded)  in usb_pc_load_mem()
692 				bus_dmamap_unload(pc->tag, pc->map);  in usb_pc_load_mem()
699 			    pc->tag, pc->map, pc->buffer, size,  in usb_pc_load_mem()
700 			    &usb_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) {  in usb_pc_load_mem()
703 		pc->isloaded = 1;  in usb_pc_load_mem()
710 			pc->tag_parent->dma_error = 0;  in usb_pc_load_mem()
711 			(pc->tag_parent->func) (pc->tag_parent);  in usb_pc_load_mem()
721 usb_pc_cpu_invalidate(struct usb_page_cache *pc)  in usb_pc_cpu_invalidate()  argument
723 	if (pc->page_offset_end == pc->page_offset_buf) {  in usb_pc_cpu_invalidate()
733 	bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_POSTREAD);  in usb_pc_cpu_invalidate()
734 	bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREREAD);  in usb_pc_cpu_invalidate()
741 usb_pc_cpu_flush(struct usb_page_cache *pc)  in usb_pc_cpu_flush()  argument
743 	if (pc->page_offset_end == pc->page_offset_buf) {  in usb_pc_cpu_flush()
747 	bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREWRITE);  in usb_pc_cpu_flush()
758 usb_pc_dmamap_create(struct usb_page_cache *pc, usb_size_t size)  in usb_pc_dmamap_create()  argument
764 	info = USB_DMATAG_TO_XROOT(pc->tag_parent);  in usb_pc_dmamap_create()
770 	utag = usb_dma_tag_find(pc->tag_parent, size, 1);  in usb_pc_dmamap_create()
775 	if (bus_dmamap_create(utag->tag, 0, &pc->map)) {  in usb_pc_dmamap_create()
778 	pc->tag = utag->tag;  in usb_pc_dmamap_create()
782 	pc->map = NULL;  in usb_pc_dmamap_create()
783 	pc->tag = NULL;  in usb_pc_dmamap_create()
793 usb_pc_dmamap_destroy(struct usb_page_cache *pc)  in usb_pc_dmamap_destroy()  argument
795 	if (pc && pc->tag) {  in usb_pc_dmamap_destroy()
796 		if (pc->isloaded)  in usb_pc_dmamap_destroy()
797 			bus_dmamap_unload(pc->tag, pc->map);  in usb_pc_dmamap_destroy()
798 		bus_dmamap_destroy(pc->tag, pc->map);  in usb_pc_dmamap_destroy()
799 		pc->tag = NULL;  in usb_pc_dmamap_destroy()
800 		pc->map = NULL;  in usb_pc_dmamap_destroy()
1058 	struct usb_page_cache *pc;  in usb_bdma_pre_sync()  local
1069 	pc = xfer->frbuffers;  in usb_bdma_pre_sync()
1072 		if (pc->isread) {  in usb_bdma_pre_sync()
1073 			usb_pc_cpu_invalidate(pc);  in usb_bdma_pre_sync()
1075 			usb_pc_cpu_flush(pc);  in usb_bdma_pre_sync()
1077 		pc++;  in usb_bdma_pre_sync()
1090 	struct usb_page_cache *pc;  in usb_bdma_post_sync()  local
1101 	pc = xfer->frbuffers;  in usb_bdma_post_sync()
1104 		if (pc->isread) {  in usb_bdma_post_sync()
1105 			usb_pc_cpu_invalidate(pc);  in usb_bdma_post_sync()
1107 		pc++;  in usb_bdma_post_sync()