33 virtio_user_create_queue(struct virtio_user_dev *dev, uint32_t queue_sel)  in virtio_user_create_queue()  argument
42 	file.fd = dev->callfds[queue_sel];  in virtio_user_create_queue()
43 	dev->ops->send_request(dev, VHOST_USER_SET_VRING_CALL, &file);  in virtio_user_create_queue()
49 virtio_user_kick_queue(struct virtio_user_dev *dev, uint32_t queue_sel)  in virtio_user_kick_queue()  argument
53 	struct vring *vring = &dev->vrings[queue_sel];  in virtio_user_kick_queue()
54 	struct vring_packed *pq_vring = &dev->packed_vrings[queue_sel];  in virtio_user_kick_queue()
61 	if (dev->features & (1ULL << VIRTIO_F_RING_PACKED)) {  in virtio_user_kick_queue()
76 	dev->ops->send_request(dev, VHOST_USER_SET_VRING_NUM, &state);  in virtio_user_kick_queue()
80 	if (dev->features & (1ULL << VIRTIO_F_RING_PACKED))  in virtio_user_kick_queue()
82 	dev->ops->send_request(dev, VHOST_USER_SET_VRING_BASE, &state);  in virtio_user_kick_queue()
84 	dev->ops->send_request(dev, VHOST_USER_SET_VRING_ADDR, &addr);  in virtio_user_kick_queue()
91 	file.fd = dev->kickfds[queue_sel];  in virtio_user_kick_queue()
92 	dev->ops->send_request(dev, VHOST_USER_SET_VRING_KICK, &file);  in virtio_user_kick_queue()
98 virtio_user_queue_setup(struct virtio_user_dev *dev,  in virtio_user_queue_setup()  argument
103 	for (i = 0; i < dev->max_queue_pairs; ++i) {  in virtio_user_queue_setup()
105 		if (fn(dev, queue_sel) < 0) {  in virtio_user_queue_setup()
110 	for (i = 0; i < dev->max_queue_pairs; ++i) {  in virtio_user_queue_setup()
112 		if (fn(dev, queue_sel) < 0) {  in virtio_user_queue_setup()
122 virtio_user_dev_set_features(struct virtio_user_dev *dev)  in virtio_user_dev_set_features()  argument
127 	pthread_mutex_lock(&dev->mutex);  in virtio_user_dev_set_features()
129 	if (dev->backend_type == VIRTIO_USER_BACKEND_VHOST_USER &&  in virtio_user_dev_set_features()
130 			dev->vhostfd < 0)  in virtio_user_dev_set_features()
134 	if (virtio_user_queue_setup(dev, virtio_user_create_queue) < 0)  in virtio_user_dev_set_features()
137 	features = dev->features;  in virtio_user_dev_set_features()
144 	ret = dev->ops->send_request(dev, VHOST_USER_SET_FEATURES, &features);  in virtio_user_dev_set_features()
149 	pthread_mutex_unlock(&dev->mutex);  in virtio_user_dev_set_features()
155 virtio_user_start_device(struct virtio_user_dev *dev)  in virtio_user_start_device()  argument
173 	pthread_mutex_lock(&dev->mutex);  in virtio_user_start_device()
175 	if (dev->backend_type == VIRTIO_USER_BACKEND_VHOST_USER &&  in virtio_user_start_device()
176 			dev->vhostfd < 0)  in virtio_user_start_device()
180 	ret = dev->ops->send_request(dev, VHOST_USER_SET_MEM_TABLE, NULL);  in virtio_user_start_device()
185 	if (virtio_user_queue_setup(dev, virtio_user_kick_queue) < 0)  in virtio_user_start_device()
191 	dev->ops->enable_qp(dev, 0, 1);  in virtio_user_start_device()
193 	dev->started = true;  in virtio_user_start_device()
194 	pthread_mutex_unlock(&dev->mutex);  in virtio_user_start_device()
199 	pthread_mutex_unlock(&dev->mutex);  in virtio_user_start_device()
205 int virtio_user_stop_device(struct virtio_user_dev *dev)  in virtio_user_stop_device()  argument
211 	pthread_mutex_lock(&dev->mutex);  in virtio_user_stop_device()
212 	if (!dev->started)  in virtio_user_stop_device()
215 	for (i = 0; i < dev->max_queue_pairs; ++i)  in virtio_user_stop_device()
216 		dev->ops->enable_qp(dev, i, 0);  in virtio_user_stop_device()
219 	for (i = 0; i < dev->max_queue_pairs * 2; ++i) {  in virtio_user_stop_device()
221 		if (dev->ops->send_request(dev, VHOST_USER_GET_VRING_BASE,  in virtio_user_stop_device()
230 	dev->started = false;  in virtio_user_stop_device()
232 	pthread_mutex_unlock(&dev->mutex);  in virtio_user_stop_device()
238 parse_mac(struct virtio_user_dev *dev, const char *mac)  in parse_mac()  argument
246 		memcpy(dev->mac_addr, &tmp, RTE_ETHER_ADDR_LEN);  in parse_mac()
247 		dev->mac_specified = 1;  in parse_mac()
255 virtio_user_dev_init_notify(struct virtio_user_dev *dev)  in virtio_user_dev_init_notify()  argument
262 		if (i >= dev->max_queue_pairs * 2) {  in virtio_user_dev_init_notify()
263 			dev->kickfds[i] = -1;  in virtio_user_dev_init_notify()
264 			dev->callfds[i] = -1;  in virtio_user_dev_init_notify()
282 		dev->callfds[i] = callfd;  in virtio_user_dev_init_notify()
283 		dev->kickfds[i] = kickfd;  in virtio_user_dev_init_notify()
288 			close(dev->callfds[j]);  in virtio_user_dev_init_notify()
289 			close(dev->kickfds[j]);  in virtio_user_dev_init_notify()
299 virtio_user_fill_intr_handle(struct virtio_user_dev *dev)  in virtio_user_fill_intr_handle()  argument
302 	struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->port_id];  in virtio_user_fill_intr_handle()
313 	for (i = 0; i < dev->max_queue_pairs; ++i)  in virtio_user_fill_intr_handle()
314 		eth_dev->intr_handle->efds[i] = dev->callfds[i];  in virtio_user_fill_intr_handle()
315 	eth_dev->intr_handle->nb_efd = dev->max_queue_pairs;  in virtio_user_fill_intr_handle()
316 	eth_dev->intr_handle->max_intr = dev->max_queue_pairs + 1;  in virtio_user_fill_intr_handle()
321 	if (dev->vhostfd >= 0)  in virtio_user_fill_intr_handle()
322 		eth_dev->intr_handle->fd = dev->vhostfd;  in virtio_user_fill_intr_handle()
323 	else if (dev->is_server)  in virtio_user_fill_intr_handle()
324 		eth_dev->intr_handle->fd = dev->listenfd;  in virtio_user_fill_intr_handle()
335 	struct virtio_user_dev *dev = arg;  in virtio_user_mem_event_cb()  local
344 	pthread_mutex_lock(&dev->mutex);  in virtio_user_mem_event_cb()
346 	if (dev->started == false)  in virtio_user_mem_event_cb()
350 	for (i = 0; i < dev->queue_pairs; i++)  in virtio_user_mem_event_cb()
351 		dev->ops->enable_qp(dev, i, 0);  in virtio_user_mem_event_cb()
354 	dev->ops->send_request(dev, VHOST_USER_SET_MEM_TABLE, NULL);  in virtio_user_mem_event_cb()
357 	for (i = 0; i < dev->queue_pairs; i++)  in virtio_user_mem_event_cb()
358 		dev->ops->enable_qp(dev, i, 1);  in virtio_user_mem_event_cb()
361 	pthread_mutex_unlock(&dev->mutex);  in virtio_user_mem_event_cb()
365 virtio_user_dev_setup(struct virtio_user_dev *dev)  in virtio_user_dev_setup()  argument
369 	dev->vhostfd = -1;  in virtio_user_dev_setup()
370 	dev->vhostfds = NULL;  in virtio_user_dev_setup()
371 	dev->tapfds = NULL;  in virtio_user_dev_setup()
373 	if (dev->is_server) {  in virtio_user_dev_setup()
374 		if (dev->backend_type != VIRTIO_USER_BACKEND_VHOST_USER) {  in virtio_user_dev_setup()
378 		dev->ops = &virtio_ops_user;  in virtio_user_dev_setup()
380 		if (dev->backend_type == VIRTIO_USER_BACKEND_VHOST_USER) {  in virtio_user_dev_setup()
381 			dev->ops = &virtio_ops_user;  in virtio_user_dev_setup()
382 		} else if (dev->backend_type ==  in virtio_user_dev_setup()
384 			dev->ops = &virtio_ops_kernel;  in virtio_user_dev_setup()
386 			dev->vhostfds = malloc(dev->max_queue_pairs *  in virtio_user_dev_setup()
388 			dev->tapfds = malloc(dev->max_queue_pairs *  in virtio_user_dev_setup()
390 			if (!dev->vhostfds || !dev->tapfds) {  in virtio_user_dev_setup()
395 			for (q = 0; q < dev->max_queue_pairs; ++q) {  in virtio_user_dev_setup()
396 				dev->vhostfds[q] = -1;  in virtio_user_dev_setup()
397 				dev->tapfds[q] = -1;  in virtio_user_dev_setup()
399 		} else if (dev->backend_type ==  in virtio_user_dev_setup()
401 			dev->ops = &virtio_ops_vdpa;  in virtio_user_dev_setup()
408 	if (dev->ops->setup(dev) < 0)  in virtio_user_dev_setup()
411 	if (virtio_user_dev_init_notify(dev) < 0)  in virtio_user_dev_setup()
414 	if (virtio_user_fill_intr_handle(dev) < 0)  in virtio_user_dev_setup()
448 virtio_user_dev_init(struct virtio_user_dev *dev, char *path, int queues,  in virtio_user_dev_init()  argument
455 	pthread_mutex_init(&dev->mutex, NULL);  in virtio_user_dev_init()
456 	strlcpy(dev->path, path, PATH_MAX);  in virtio_user_dev_init()
457 	dev->started = 0;  in virtio_user_dev_init()
458 	dev->max_queue_pairs = queues;  in virtio_user_dev_init()
459 	dev->queue_pairs = 1; /* mq disabled by default */  in virtio_user_dev_init()
460 	dev->queue_size = queue_size;  in virtio_user_dev_init()
461 	dev->is_server = server;  in virtio_user_dev_init()
462 	dev->mac_specified = 0;  in virtio_user_dev_init()
463 	dev->frontend_features = 0;  in virtio_user_dev_init()
464 	dev->unsupported_features = ~VIRTIO_USER_SUPPORTED_FEATURES;  in virtio_user_dev_init()
465 	dev->protocol_features = VIRTIO_USER_SUPPORTED_PROTOCOL_FEATURES;  in virtio_user_dev_init()
466 	dev->backend_type = backend_type;  in virtio_user_dev_init()
468 	parse_mac(dev, mac);  in virtio_user_dev_init()
471 		dev->ifname = *ifname;  in virtio_user_dev_init()
475 	if (virtio_user_dev_setup(dev) < 0) {  in virtio_user_dev_init()
480 	if (dev->backend_type != VIRTIO_USER_BACKEND_VHOST_USER)  in virtio_user_dev_init()
481 		dev->unsupported_features |=  in virtio_user_dev_init()
484 	if (!dev->is_server) {  in virtio_user_dev_init()
485 		if (dev->ops->send_request(dev, VHOST_USER_SET_OWNER,  in virtio_user_dev_init()
492 		if (dev->ops->send_request(dev, VHOST_USER_GET_FEATURES,  in virtio_user_dev_init()
493 					   &dev->device_features) < 0) {  in virtio_user_dev_init()
500 		if (dev->device_features &  in virtio_user_dev_init()
502 			if (dev->ops->send_request(dev,  in virtio_user_dev_init()
507 			dev->protocol_features &= protocol_features;  in virtio_user_dev_init()
509 			if (dev->ops->send_request(dev,  in virtio_user_dev_init()
511 					&dev->protocol_features))  in virtio_user_dev_init()
514 			if (!(dev->protocol_features &  in virtio_user_dev_init()
516 				dev->unsupported_features |=  in virtio_user_dev_init()
525 		dev->device_features = VIRTIO_USER_SUPPORTED_FEATURES;  in virtio_user_dev_init()
530 		dev->protocol_features &=  in virtio_user_dev_init()
537 		dev->unsupported_features |= (1ull << VIRTIO_NET_F_MRG_RXBUF);  in virtio_user_dev_init()
540 		dev->unsupported_features |= (1ull << VIRTIO_F_IN_ORDER);  in virtio_user_dev_init()
543 		dev->unsupported_features |= (1ull << VIRTIO_F_RING_PACKED);  in virtio_user_dev_init()
545 	if (dev->mac_specified)  in virtio_user_dev_init()
546 		dev->frontend_features |= (1ull << VIRTIO_NET_F_MAC);  in virtio_user_dev_init()
548 		dev->unsupported_features |= (1ull << VIRTIO_NET_F_MAC);  in virtio_user_dev_init()
554 		dev->frontend_features |= (1ull << VIRTIO_NET_F_CTRL_VQ);  in virtio_user_dev_init()
556 		dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_VQ);  in virtio_user_dev_init()
558 		dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_RX);  in virtio_user_dev_init()
559 		dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_VLAN);  in virtio_user_dev_init()
560 		dev->unsupported_features |=  in virtio_user_dev_init()
562 		dev->unsupported_features |= (1ull << VIRTIO_NET_F_MQ);  in virtio_user_dev_init()
563 		dev->unsupported_features |=  in virtio_user_dev_init()
568 	if (dev->backend_type == VIRTIO_USER_BACKEND_VHOST_USER)  in virtio_user_dev_init()
569 		dev->frontend_features |= (1ull << VIRTIO_NET_F_STATUS);  in virtio_user_dev_init()
575 	dev->device_features |= dev->frontend_features;  in virtio_user_dev_init()
576 	dev->device_features &= ~dev->unsupported_features;  in virtio_user_dev_init()
579 				virtio_user_mem_event_cb, dev)) {  in virtio_user_dev_init()
591 virtio_user_dev_uninit(struct virtio_user_dev *dev)  in virtio_user_dev_uninit()  argument
595 	virtio_user_stop_device(dev);  in virtio_user_dev_uninit()
597 	rte_mem_event_callback_unregister(VIRTIO_USER_MEM_EVENT_CLB_NAME, dev);  in virtio_user_dev_uninit()
599 	for (i = 0; i < dev->max_queue_pairs * 2; ++i) {  in virtio_user_dev_uninit()
600 		close(dev->callfds[i]);  in virtio_user_dev_uninit()
601 		close(dev->kickfds[i]);  in virtio_user_dev_uninit()
604 	if (dev->vhostfd >= 0)  in virtio_user_dev_uninit()
605 		close(dev->vhostfd);  in virtio_user_dev_uninit()
607 	if (dev->is_server && dev->listenfd >= 0) {  in virtio_user_dev_uninit()
608 		close(dev->listenfd);  in virtio_user_dev_uninit()
609 		dev->listenfd = -1;  in virtio_user_dev_uninit()
612 	if (dev->vhostfds) {  in virtio_user_dev_uninit()
613 		for (i = 0; i < dev->max_queue_pairs; ++i) {  in virtio_user_dev_uninit()
614 			close(dev->vhostfds[i]);  in virtio_user_dev_uninit()
615 			if (dev->tapfds[i] >= 0)  in virtio_user_dev_uninit()
616 				close(dev->tapfds[i]);  in virtio_user_dev_uninit()
618 		free(dev->vhostfds);  in virtio_user_dev_uninit()
619 		free(dev->tapfds);  in virtio_user_dev_uninit()
622 	free(dev->ifname);  in virtio_user_dev_uninit()
624 	if (dev->is_server)  in virtio_user_dev_uninit()
625 		unlink(dev->path);  in virtio_user_dev_uninit()
629 virtio_user_handle_mq(struct virtio_user_dev *dev, uint16_t q_pairs)  in virtio_user_handle_mq()  argument
634 	if (q_pairs > dev->max_queue_pairs) {  in virtio_user_handle_mq()
636 			     q_pairs, dev->max_queue_pairs);  in virtio_user_handle_mq()
643 	if (!dev->is_server || dev->vhostfd >= 0) {  in virtio_user_handle_mq()
645 			ret |= dev->ops->enable_qp(dev, i, 1);  in virtio_user_handle_mq()
646 		for (i = q_pairs; i < dev->max_queue_pairs; ++i)  in virtio_user_handle_mq()
647 			ret |= dev->ops->enable_qp(dev, i, 0);  in virtio_user_handle_mq()
649 	dev->queue_pairs = q_pairs;  in virtio_user_handle_mq()
655 virtio_user_handle_ctrl_msg(struct virtio_user_dev *dev, struct vring *vring,  in virtio_user_handle_ctrl_msg()  argument
683 		status = virtio_user_handle_mq(dev, queues);  in virtio_user_handle_ctrl_msg()
706 virtio_user_handle_ctrl_msg_packed(struct virtio_user_dev *dev,  in virtio_user_handle_ctrl_msg_packed()  argument
718 	if (idx_data >= dev->queue_size)  in virtio_user_handle_ctrl_msg_packed()
719 		idx_data -= dev->queue_size;  in virtio_user_handle_ctrl_msg_packed()
726 		if (idx_status >= dev->queue_size)  in virtio_user_handle_ctrl_msg_packed()
727 			idx_status -= dev->queue_size;  in virtio_user_handle_ctrl_msg_packed()
738 		status = virtio_user_handle_mq(dev, queues);  in virtio_user_handle_ctrl_msg_packed()
757 virtio_user_handle_cq_packed(struct virtio_user_dev *dev, uint16_t queue_idx)  in virtio_user_handle_cq_packed()  argument
759 	struct virtio_user_queue *vq = &dev->packed_queues[queue_idx];  in virtio_user_handle_cq_packed()
760 	struct vring_packed *vring = &dev->packed_vrings[queue_idx];  in virtio_user_handle_cq_packed()
770 		n_descs = virtio_user_handle_ctrl_msg_packed(dev, vring,  in virtio_user_handle_cq_packed()
781 		if (vq->used_idx >= dev->queue_size) {  in virtio_user_handle_cq_packed()
782 			vq->used_idx -= dev->queue_size;  in virtio_user_handle_cq_packed()
789 virtio_user_handle_cq(struct virtio_user_dev *dev, uint16_t queue_idx)  in virtio_user_handle_cq()  argument
794 	struct vring *vring = &dev->vrings[queue_idx];  in virtio_user_handle_cq()
803 		n_descs = virtio_user_handle_ctrl_msg(dev, vring, desc_idx);  in virtio_user_handle_cq()
815 virtio_user_dev_set_status(struct virtio_user_dev *dev, uint8_t status)  in virtio_user_dev_set_status()  argument
820 	pthread_mutex_lock(&dev->mutex);  in virtio_user_dev_set_status()
821 	dev->status = status;  in virtio_user_dev_set_status()
822 	if (dev->backend_type == VIRTIO_USER_BACKEND_VHOST_USER)  in virtio_user_dev_set_status()
823 		ret = dev->ops->send_request(dev,  in virtio_user_dev_set_status()
825 	else if (dev->backend_type == VIRTIO_USER_BACKEND_VHOST_VDPA)  in virtio_user_dev_set_status()
826 		ret = dev->ops->send_request(dev,  in virtio_user_dev_set_status()
836 	pthread_mutex_unlock(&dev->mutex);  in virtio_user_dev_set_status()
841 virtio_user_dev_update_status(struct virtio_user_dev *dev)  in virtio_user_dev_update_status()  argument
847 	pthread_mutex_lock(&dev->mutex);  in virtio_user_dev_update_status()
848 	if (dev->backend_type == VIRTIO_USER_BACKEND_VHOST_USER) {  in virtio_user_dev_update_status()
849 		err = dev->ops->send_request(dev, VHOST_USER_GET_STATUS, &ret);  in virtio_user_dev_update_status()
858 	} else if (dev->backend_type == VIRTIO_USER_BACKEND_VHOST_VDPA) {  in virtio_user_dev_update_status()
859 		err = dev->ops->send_request(dev, VHOST_USER_GET_STATUS,  in virtio_user_dev_update_status()
866 		dev->status = status;  in virtio_user_dev_update_status()
875 			dev->status,  in virtio_user_dev_update_status()
876 			(dev->status == VIRTIO_CONFIG_STATUS_RESET),  in virtio_user_dev_update_status()
877 			!!(dev->status & VIRTIO_CONFIG_STATUS_ACK),  in virtio_user_dev_update_status()
878 			!!(dev->status & VIRTIO_CONFIG_STATUS_DRIVER),  in virtio_user_dev_update_status()
879 			!!(dev->status & VIRTIO_CONFIG_STATUS_DRIVER_OK),  in virtio_user_dev_update_status()
880 			!!(dev->status & VIRTIO_CONFIG_STATUS_FEATURES_OK),  in virtio_user_dev_update_status()
881 			!!(dev->status & VIRTIO_CONFIG_STATUS_DEV_NEED_RESET),  in virtio_user_dev_update_status()
882 			!!(dev->status & VIRTIO_CONFIG_STATUS_FAILED));  in virtio_user_dev_update_status()
889 	pthread_mutex_unlock(&dev->mutex);  in virtio_user_dev_update_status()