64 gpa_to_vva(struct vhost_blk_ctrlr *ctrlr, uint64_t gpa, uint64_t *len)  in gpa_to_vva()  argument
66 	assert(ctrlr->mem != NULL);  in gpa_to_vva()
68 	return rte_vhost_va_from_guest_pa(ctrlr->mem, gpa, len);  in gpa_to_vva()
77 	rte_vhost_set_last_inflight_io_split(task->ctrlr->vid,  in enqueue_task()
93 	rte_vhost_clr_inflight_desc_split(task->ctrlr->vid,  in enqueue_task()
99 	rte_vhost_vring_call(task->ctrlr->vid, vq->id);  in enqueue_task()
108 	rte_vhost_set_last_inflight_io_packed(task->ctrlr->vid, vq->id,  in enqueue_task_packed()
122 	rte_vhost_clr_inflight_desc_packed(task->ctrlr->vid, vq->id,  in enqueue_task_packed()
134 	rte_vhost_vring_call(task->ctrlr->vid, vq->id);  in enqueue_task_packed()
150 desc_payload_to_iovs(struct vhost_blk_ctrlr *ctrlr, struct iovec *iovs,  in desc_payload_to_iovs()  argument
162 		vva = (void *)(uintptr_t)gpa_to_vva(ctrlr,  in desc_payload_to_iovs()
210 setup_iovs_from_descs_split(struct vhost_blk_ctrlr *ctrlr,  in setup_iovs_from_descs_split()  argument
226 		if (desc_payload_to_iovs(ctrlr, iovs, iovs_idx,  in setup_iovs_from_descs_split()
241 setup_iovs_from_descs_packed(struct vhost_blk_ctrlr *ctrlr,  in setup_iovs_from_descs_packed()  argument
257 		if (desc_payload_to_iovs(ctrlr, iovs, iovs_idx,  in setup_iovs_from_descs_packed()
272 setup_iovs_from_inflight_desc(struct vhost_blk_ctrlr *ctrlr,  in setup_iovs_from_inflight_desc()  argument
292 		if (desc_payload_to_iovs(ctrlr, iovs, iovs_idx,  in setup_iovs_from_inflight_desc()
320 			if (setup_iovs_from_inflight_desc(task->ctrlr, task->vq,  in process_blk_task()
327 			if (setup_iovs_from_descs_packed(task->ctrlr, task->vq,  in process_blk_task()
335 		if (setup_iovs_from_descs_split(task->ctrlr, task->vq,  in process_blk_task()
354 	if (vhost_bdev_process_blk_commands(task->ctrlr->bdev, task))  in process_blk_task()
507 			rte_vhost_set_inflight_desc_packed(task->ctrlr->vid,  in process_vq()
523 			rte_vhost_set_inflight_desc_split(task->ctrlr->vid,  in process_vq()
534 	struct vhost_blk_ctrlr *ctrlr = (struct vhost_blk_ctrlr *)arg;  in ctrlr_worker()  local
541 	if (ctrlr == NULL || ctrlr->bdev == NULL) {  in ctrlr_worker()
554 		submit_inflight_vq(&ctrlr->queues[i]);  in ctrlr_worker()
558 			process_vq(&ctrlr->queues[i]);  in ctrlr_worker()
566 alloc_task_pool(struct vhost_blk_ctrlr *ctrlr)  in alloc_task_pool()  argument
572 		vq = &ctrlr->queues[i];  in alloc_task_pool()
583 			vq->tasks[j].ctrlr = ctrlr;  in alloc_task_pool()
592 free_task_pool(struct vhost_blk_ctrlr *ctrlr)  in free_task_pool()  argument
597 		rte_free(ctrlr->queues[i].tasks);  in free_task_pool()
603 	struct vhost_blk_ctrlr *ctrlr;  in new_device()  local
617 	ctrlr = vhost_blk_ctrlr_find(path);  in new_device()
618 	if (!ctrlr) {  in new_device()
623 	if (ctrlr->started)  in new_device()
626 	ctrlr->vid = vid;  in new_device()
636 		vq = &ctrlr->queues[i];  in new_device()
639 		assert(rte_vhost_get_vhost_vring(ctrlr->vid, i,  in new_device()
641 		assert(rte_vhost_get_vring_base(ctrlr->vid, i,  in new_device()
644 		assert(rte_vhost_get_vhost_ring_inflight(ctrlr->vid, i,  in new_device()
650 				ctrlr->vid, i,  in new_device()
668 	assert(rte_vhost_get_mem_table(vid, &ctrlr->mem) == 0);  in new_device()
669 	assert(ctrlr->mem != NULL);  in new_device()
670 	assert(alloc_task_pool(ctrlr) == 0);  in new_device()
675 	if (pthread_create(&tid, NULL, &ctrlr_worker, ctrlr) < 0) {  in new_device()
681 	ctrlr->started = 1;  in new_device()
690 	struct vhost_blk_ctrlr *ctrlr;  in destroy_device()  local
701 	ctrlr = vhost_blk_ctrlr_find(path);  in destroy_device()
702 	if (!ctrlr) {  in destroy_device()
707 	if (!ctrlr->started)  in destroy_device()
714 		vq = &ctrlr->queues[i];  in destroy_device()
722 		rte_vhost_set_vring_base(ctrlr->vid, i,  in destroy_device()
727 	free_task_pool(ctrlr);  in destroy_device()
728 	free(ctrlr->mem);  in destroy_device()
730 	ctrlr->started = 0;  in destroy_device()
784 	struct vhost_blk_ctrlr *ctrlr;  in vhost_blk_ctrlr_construct()  local
813 	ctrlr = rte_zmalloc(NULL, sizeof(*ctrlr), RTE_CACHE_LINE_SIZE);  in vhost_blk_ctrlr_construct()
814 	if (!ctrlr) {  in vhost_blk_ctrlr_construct()
820 	ctrlr->bdev = vhost_blk_bdev_construct("malloc0", "vhost_blk_malloc0",  in vhost_blk_ctrlr_construct()
822 	if (!ctrlr->bdev) {  in vhost_blk_ctrlr_construct()
823 		rte_free(ctrlr);  in vhost_blk_ctrlr_construct()
831 	return ctrlr;  in vhost_blk_ctrlr_construct()
835 vhost_blk_ctrlr_destroy(struct vhost_blk_ctrlr *ctrlr)  in vhost_blk_ctrlr_destroy()  argument
837 	if (ctrlr->bdev != NULL) {  in vhost_blk_ctrlr_destroy()
838 		if (ctrlr->bdev->data != NULL)  in vhost_blk_ctrlr_destroy()
839 			rte_free(ctrlr->bdev->data);  in vhost_blk_ctrlr_destroy()
841 		rte_free(ctrlr->bdev);  in vhost_blk_ctrlr_destroy()
843 	rte_free(ctrlr);  in vhost_blk_ctrlr_destroy()
851 	struct vhost_blk_ctrlr *ctrlr;  in signal_handler()  local
853 	ctrlr = vhost_blk_ctrlr_find(dev_pathname);  in signal_handler()
854 	if (ctrlr == NULL)  in signal_handler()
857 	if (ctrlr->started)  in signal_handler()
858 		destroy_device(ctrlr->vid);  in signal_handler()
860 	vhost_blk_ctrlr_destroy(ctrlr);  in signal_handler()