94 static int send_vhost_reply(int sockfd, struct VhostUserMsg *msg);
95 static int read_vhost_message(int sockfd, struct VhostUserMsg *msg);
98 close_msg_fds(struct VhostUserMsg *msg)  in close_msg_fds()  argument
102 	for (i = 0; i < msg->fd_num; i++) {  in close_msg_fds()
103 		int fd = msg->fds[i];  in close_msg_fds()
108 		msg->fds[i] = -1;  in close_msg_fds()
118 validate_msg_fds(struct VhostUserMsg *msg, int expected_fds)  in validate_msg_fds()  argument
120 	if (msg->fd_num == expected_fds)  in validate_msg_fds()
126 		vhost_message_str[msg->request.master],  in validate_msg_fds()
127 		msg->fd_num);  in validate_msg_fds()
129 	close_msg_fds(msg);  in validate_msg_fds()
233 			struct VhostUserMsg *msg,  in vhost_user_set_owner()  argument
236 	if (validate_msg_fds(msg, 0) != 0)  in vhost_user_set_owner()
244 			struct VhostUserMsg *msg,  in vhost_user_reset_owner()  argument
249 	if (validate_msg_fds(msg, 0) != 0)  in vhost_user_reset_owner()
263 vhost_user_get_features(struct virtio_net **pdev, struct VhostUserMsg *msg,  in vhost_user_get_features()  argument
269 	if (validate_msg_fds(msg, 0) != 0)  in vhost_user_get_features()
274 	msg->payload.u64 = features;  in vhost_user_get_features()
275 	msg->size = sizeof(msg->payload.u64);  in vhost_user_get_features()
276 	msg->fd_num = 0;  in vhost_user_get_features()
285 vhost_user_get_queue_num(struct virtio_net **pdev, struct VhostUserMsg *msg,  in vhost_user_get_queue_num()  argument
291 	if (validate_msg_fds(msg, 0) != 0)  in vhost_user_get_queue_num()
296 	msg->payload.u64 = (uint64_t)queue_num;  in vhost_user_get_queue_num()
297 	msg->size = sizeof(msg->payload.u64);  in vhost_user_get_queue_num()
298 	msg->fd_num = 0;  in vhost_user_get_queue_num()
307 vhost_user_set_features(struct virtio_net **pdev, struct VhostUserMsg *msg,  in vhost_user_set_features()  argument
311 	uint64_t features = msg->payload.u64;  in vhost_user_set_features()
315 	if (validate_msg_fds(msg, 0) != 0)  in vhost_user_set_features()
400 			struct VhostUserMsg *msg,  in vhost_user_set_vring_num()  argument
404 	struct vhost_virtqueue *vq = dev->virtqueue[msg->payload.state.index];  in vhost_user_set_vring_num()
406 	if (validate_msg_fds(msg, 0) != 0)  in vhost_user_set_vring_num()
409 	vq->size = msg->payload.state.num;  in vhost_user_set_vring_num()
793 vhost_user_set_vring_addr(struct virtio_net **pdev, struct VhostUserMsg *msg,  in vhost_user_set_vring_addr()  argument
798 	struct vhost_vring_addr *addr = &msg->payload.addr;  in vhost_user_set_vring_addr()
801 	if (validate_msg_fds(msg, 0) != 0)  in vhost_user_set_vring_addr()
808 	vq = dev->virtqueue[msg->payload.addr.index];  in vhost_user_set_vring_addr()
823 		dev = translate_ring_addresses(dev, msg->payload.addr.index);  in vhost_user_set_vring_addr()
838 			struct VhostUserMsg *msg,  in vhost_user_set_vring_base()  argument
842 	struct vhost_virtqueue *vq = dev->virtqueue[msg->payload.state.index];  in vhost_user_set_vring_base()
843 	uint64_t val = msg->payload.state.num;  in vhost_user_set_vring_base()
845 	if (validate_msg_fds(msg, 0) != 0)  in vhost_user_set_vring_base()
863 		vq->last_used_idx = msg->payload.state.num;  in vhost_user_set_vring_base()
864 		vq->last_avail_idx = msg->payload.state.num;  in vhost_user_set_vring_base()
1002 vhost_user_set_mem_table(struct virtio_net **pdev, struct VhostUserMsg *msg,  in vhost_user_set_mem_table()  argument
1006 	struct VhostUserMemory *memory = &msg->payload.memory;  in vhost_user_set_mem_table()
1015 	if (validate_msg_fds(msg, memory->nregions) != 0)  in vhost_user_set_mem_table()
1028 		close_msg_fds(msg);  in vhost_user_set_mem_table()
1083 		reg->fd              = msg->fds[i];  in vhost_user_set_mem_table()
1089 		msg->fds[i] = -1;  in vhost_user_set_mem_table()
1188 		msg->fd_num = 0;  in vhost_user_set_mem_table()
1189 		send_vhost_reply(main_fd, msg);  in vhost_user_set_mem_table()
1282 	close_msg_fds(msg);  in vhost_user_set_mem_table()
1409 			   VhostUserMsg *msg,  in vhost_user_get_inflight_fd()  argument
1419 	if (msg->size != sizeof(msg->payload.inflight)) {  in vhost_user_get_inflight_fd()
1422 			msg->size);  in vhost_user_get_inflight_fd()
1437 	num_queues = msg->payload.inflight.num_queues;  in vhost_user_get_inflight_fd()
1438 	queue_size = msg->payload.inflight.queue_size;  in vhost_user_get_inflight_fd()
1441 		msg->payload.inflight.num_queues);  in vhost_user_get_inflight_fd()
1443 		msg->payload.inflight.queue_size);  in vhost_user_get_inflight_fd()
1455 			msg->payload.inflight.mmap_size = 0;  in vhost_user_get_inflight_fd()
1471 	dev->inflight_info->size = msg->payload.inflight.mmap_size = mmap_size;  in vhost_user_get_inflight_fd()
1472 	dev->inflight_info->fd = msg->fds[0] = fd;  in vhost_user_get_inflight_fd()
1473 	msg->payload.inflight.mmap_offset = 0;  in vhost_user_get_inflight_fd()
1474 	msg->fd_num = 1;  in vhost_user_get_inflight_fd()
1490 		msg->payload.inflight.mmap_size);  in vhost_user_get_inflight_fd()
1493 		msg->payload.inflight.mmap_offset);  in vhost_user_get_inflight_fd()
1495 		"send inflight fd: %d\n", msg->fds[0]);  in vhost_user_get_inflight_fd()
1501 vhost_user_set_inflight_fd(struct virtio_net **pdev, VhostUserMsg *msg,  in vhost_user_set_inflight_fd()  argument
1512 	fd = msg->fds[0];  in vhost_user_set_inflight_fd()
1513 	if (msg->size != sizeof(msg->payload.inflight) || fd < 0) {  in vhost_user_set_inflight_fd()
1516 			msg->size, fd);  in vhost_user_set_inflight_fd()
1520 	mmap_size = msg->payload.inflight.mmap_size;  in vhost_user_set_inflight_fd()
1521 	mmap_offset = msg->payload.inflight.mmap_offset;  in vhost_user_set_inflight_fd()
1522 	num_queues = msg->payload.inflight.num_queues;  in vhost_user_set_inflight_fd()
1523 	queue_size = msg->payload.inflight.queue_size;  in vhost_user_set_inflight_fd()
1595 vhost_user_set_vring_call(struct virtio_net **pdev, struct VhostUserMsg *msg,  in vhost_user_set_vring_call()  argument
1603 	expected_fds = (msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK) ? 0 : 1;  in vhost_user_set_vring_call()
1604 	if (validate_msg_fds(msg, expected_fds) != 0)  in vhost_user_set_vring_call()
1607 	file.index = msg->payload.u64 & VHOST_USER_VRING_IDX_MASK;  in vhost_user_set_vring_call()
1608 	if (msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK)  in vhost_user_set_vring_call()
1611 		file.fd = msg->fds[0];  in vhost_user_set_vring_call()
1631 			struct VhostUserMsg *msg,  in vhost_user_set_vring_err()  argument
1636 	expected_fds = (msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK) ? 0 : 1;  in vhost_user_set_vring_err()
1637 	if (validate_msg_fds(msg, expected_fds) != 0)  in vhost_user_set_vring_err()
1640 	if (!(msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK))  in vhost_user_set_vring_err()
1641 		close(msg->fds[0]);  in vhost_user_set_vring_err()
1839 vhost_user_set_vring_kick(struct virtio_net **pdev, struct VhostUserMsg *msg,  in vhost_user_set_vring_kick()  argument
1847 	expected_fds = (msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK) ? 0 : 1;  in vhost_user_set_vring_kick()
1848 	if (validate_msg_fds(msg, expected_fds) != 0)  in vhost_user_set_vring_kick()
1851 	file.index = msg->payload.u64 & VHOST_USER_VRING_IDX_MASK;  in vhost_user_set_vring_kick()
1852 	if (msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK)  in vhost_user_set_vring_kick()
1855 		file.fd = msg->fds[0];  in vhost_user_set_vring_kick()
1915 			struct VhostUserMsg *msg,  in vhost_user_get_vring_base()  argument
1919 	struct vhost_virtqueue *vq = dev->virtqueue[msg->payload.state.index];  in vhost_user_get_vring_base()
1922 	if (validate_msg_fds(msg, 0) != 0)  in vhost_user_get_vring_base()
1939 		msg->payload.state.num = val;  in vhost_user_get_vring_base()
1941 		msg->payload.state.num = vq->last_avail_idx;  in vhost_user_get_vring_base()
1945 		"vring base idx:%d file:%d\n", msg->payload.state.index,  in vhost_user_get_vring_base()
1946 		msg->payload.state.num);  in vhost_user_get_vring_base()
1981 	msg->size = sizeof(msg->payload.state);  in vhost_user_get_vring_base()
1982 	msg->fd_num = 0;  in vhost_user_get_vring_base()
1995 			struct VhostUserMsg *msg,  in vhost_user_set_vring_enable()  argument
1999 	int enable = (int)msg->payload.state.num;  in vhost_user_set_vring_enable()
2000 	int index = (int)msg->payload.state.index;  in vhost_user_set_vring_enable()
2002 	if (validate_msg_fds(msg, 0) != 0)  in vhost_user_set_vring_enable()
2024 			struct VhostUserMsg *msg,  in vhost_user_get_protocol_features()  argument
2030 	if (validate_msg_fds(msg, 0) != 0)  in vhost_user_get_protocol_features()
2036 	msg->payload.u64 = protocol_features;  in vhost_user_get_protocol_features()
2037 	msg->size = sizeof(msg->payload.u64);  in vhost_user_get_protocol_features()
2038 	msg->fd_num = 0;  in vhost_user_get_protocol_features()
2045 			struct VhostUserMsg *msg,  in vhost_user_set_protocol_features()  argument
2049 	uint64_t protocol_features = msg->payload.u64;  in vhost_user_set_protocol_features()
2052 	if (validate_msg_fds(msg, 0) != 0)  in vhost_user_set_protocol_features()
2073 vhost_user_set_log_base(struct virtio_net **pdev, struct VhostUserMsg *msg,  in vhost_user_set_log_base()  argument
2077 	int fd = msg->fds[0];  in vhost_user_set_log_base()
2081 	if (validate_msg_fds(msg, 1) != 0)  in vhost_user_set_log_base()
2089 	if (msg->size != sizeof(VhostUserLog)) {  in vhost_user_set_log_base()
2092 			msg->size, (int)sizeof(VhostUserLog));  in vhost_user_set_log_base()
2096 	size = msg->payload.log.mmap_size;  in vhost_user_set_log_base()
2097 	off  = msg->payload.log.mmap_offset;  in vhost_user_set_log_base()
2137 	msg->size = 0;  in vhost_user_set_log_base()
2138 	msg->fd_num = 0;  in vhost_user_set_log_base()
2143 	close_msg_fds(msg);  in vhost_user_set_log_base()
2148 			struct VhostUserMsg *msg,  in vhost_user_set_log_fd()  argument
2151 	if (validate_msg_fds(msg, 1) != 0)  in vhost_user_set_log_fd()
2154 	close(msg->fds[0]);  in vhost_user_set_log_fd()
2169 vhost_user_send_rarp(struct virtio_net **pdev, struct VhostUserMsg *msg,  in vhost_user_send_rarp()  argument
2173 	uint8_t *mac = (uint8_t *)&msg->payload.u64;  in vhost_user_send_rarp()
2176 	if (validate_msg_fds(msg, 0) != 0)  in vhost_user_send_rarp()
2200 vhost_user_net_set_mtu(struct virtio_net **pdev, struct VhostUserMsg *msg,  in vhost_user_net_set_mtu()  argument
2205 	if (validate_msg_fds(msg, 0) != 0)  in vhost_user_net_set_mtu()
2208 	if (msg->payload.u64 < VIRTIO_MIN_MTU ||  in vhost_user_net_set_mtu()
2209 			msg->payload.u64 > VIRTIO_MAX_MTU) {  in vhost_user_net_set_mtu()
2211 				msg->payload.u64);  in vhost_user_net_set_mtu()
2216 	dev->mtu = msg->payload.u64;  in vhost_user_net_set_mtu()
2222 vhost_user_set_req_fd(struct virtio_net **pdev, struct VhostUserMsg *msg,  in vhost_user_set_req_fd()  argument
2226 	int fd = msg->fds[0];  in vhost_user_set_req_fd()
2228 	if (validate_msg_fds(msg, 1) != 0)  in vhost_user_set_req_fd()
2322 vhost_user_iotlb_msg(struct virtio_net **pdev, struct VhostUserMsg *msg,  in vhost_user_iotlb_msg()  argument
2326 	struct vhost_iotlb_msg *imsg = &msg->payload.iotlb;  in vhost_user_iotlb_msg()
2330 	if (validate_msg_fds(msg, 0) != 0)  in vhost_user_iotlb_msg()
2378 			struct VhostUserMsg *msg,  in vhost_user_set_postcopy_advise()  argument
2385 	if (validate_msg_fds(msg, 0) != 0)  in vhost_user_set_postcopy_advise()
2404 	msg->fds[0] = dev->postcopy_ufd;  in vhost_user_set_postcopy_advise()
2405 	msg->fd_num = 1;  in vhost_user_set_postcopy_advise()
2410 	msg->fd_num = 0;  in vhost_user_set_postcopy_advise()
2418 			struct VhostUserMsg *msg __rte_unused,  in vhost_user_set_postcopy_listen()
2423 	if (validate_msg_fds(msg, 0) != 0)  in vhost_user_set_postcopy_listen()
2437 vhost_user_postcopy_end(struct virtio_net **pdev, struct VhostUserMsg *msg,  in vhost_user_postcopy_end()  argument
2442 	if (validate_msg_fds(msg, 0) != 0)  in vhost_user_postcopy_end()
2451 	msg->payload.u64 = 0;  in vhost_user_postcopy_end()
2452 	msg->size = sizeof(msg->payload.u64);  in vhost_user_postcopy_end()
2453 	msg->fd_num = 0;  in vhost_user_postcopy_end()
2459 vhost_user_get_status(struct virtio_net **pdev, struct VhostUserMsg *msg,  in vhost_user_get_status()  argument
2464 	if (validate_msg_fds(msg, 0) != 0)  in vhost_user_get_status()
2467 	msg->payload.u64 = dev->status;  in vhost_user_get_status()
2468 	msg->size = sizeof(msg->payload.u64);  in vhost_user_get_status()
2469 	msg->fd_num = 0;  in vhost_user_get_status()
2475 vhost_user_set_status(struct virtio_net **pdev, struct VhostUserMsg *msg,  in vhost_user_set_status()  argument
2480 	if (validate_msg_fds(msg, 0) != 0)  in vhost_user_set_status()
2484 	if (msg->payload.u64 > UINT8_MAX) {  in vhost_user_set_status()
2486 				msg->payload.u64);  in vhost_user_set_status()
2490 	dev->status = msg->payload.u64;  in vhost_user_set_status()
2523 					struct VhostUserMsg *msg,
2560 read_vhost_message(int sockfd, struct VhostUserMsg *msg)  in read_vhost_message()  argument
2564 	ret = read_fd_message(sockfd, (char *)msg, VHOST_USER_HDR_SIZE,  in read_vhost_message()
2565 		msg->fds, VHOST_MEMORY_MAX_NREGIONS, &msg->fd_num);  in read_vhost_message()
2570 		close_msg_fds(msg);  in read_vhost_message()
2574 	if (msg->size) {  in read_vhost_message()
2575 		if (msg->size > sizeof(msg->payload)) {  in read_vhost_message()
2577 				"invalid msg size: %d\n", msg->size);  in read_vhost_message()
2580 		ret = read(sockfd, &msg->payload, msg->size);  in read_vhost_message()
2583 		if (ret != (int)msg->size) {  in read_vhost_message()
2594 send_vhost_message(int sockfd, struct VhostUserMsg *msg)  in send_vhost_message()  argument
2596 	if (!msg)  in send_vhost_message()
2599 	return send_fd_message(sockfd, (char *)msg,  in send_vhost_message()
2600 		VHOST_USER_HDR_SIZE + msg->size, msg->fds, msg->fd_num);  in send_vhost_message()
2604 send_vhost_reply(int sockfd, struct VhostUserMsg *msg)  in send_vhost_reply()  argument
2606 	if (!msg)  in send_vhost_reply()
2609 	msg->flags &= ~VHOST_USER_VERSION_MASK;  in send_vhost_reply()
2610 	msg->flags &= ~VHOST_USER_NEED_REPLY;  in send_vhost_reply()
2611 	msg->flags |= VHOST_USER_VERSION;  in send_vhost_reply()
2612 	msg->flags |= VHOST_USER_REPLY_MASK;  in send_vhost_reply()
2614 	return send_vhost_message(sockfd, msg);  in send_vhost_reply()
2618 send_vhost_slave_message(struct virtio_net *dev, struct VhostUserMsg *msg)  in send_vhost_slave_message()  argument
2622 	if (msg->flags & VHOST_USER_NEED_REPLY)  in send_vhost_slave_message()
2625 	ret = send_vhost_message(dev->slave_req_fd, msg);  in send_vhost_slave_message()
2626 	if (ret < 0 && (msg->flags & VHOST_USER_NEED_REPLY))  in send_vhost_slave_message()
2637 			struct VhostUserMsg *msg)  in vhost_user_check_and_alloc_queue_pair()  argument
2641 	switch (msg->request.master) {  in vhost_user_check_and_alloc_queue_pair()
2645 		vring_idx = msg->payload.u64 & VHOST_USER_VRING_IDX_MASK;  in vhost_user_check_and_alloc_queue_pair()
2650 		vring_idx = msg->payload.state.index;  in vhost_user_check_and_alloc_queue_pair()
2653 		vring_idx = msg->payload.addr.index;  in vhost_user_check_and_alloc_queue_pair()
2709 	struct VhostUserMsg msg;  in vhost_user_msg_handler()  local
2731 	ret = read_vhost_message(fd, &msg);  in vhost_user_msg_handler()
2744 	request = msg.request.master;  in vhost_user_msg_handler()
2757 	ret = vhost_user_check_and_alloc_queue_pair(dev, &msg);  in vhost_user_msg_handler()
2801 				(void *)&msg);  in vhost_user_msg_handler()
2804 			send_vhost_reply(fd, &msg);  in vhost_user_msg_handler()
2819 		ret = vhost_message_handlers[request](&dev, &msg, fd);  in vhost_user_msg_handler()
2838 			send_vhost_reply(fd, &msg);  in vhost_user_msg_handler()
2850 				(void *)&msg);  in vhost_user_msg_handler()
2853 			send_vhost_reply(fd, &msg);  in vhost_user_msg_handler()
2871 		close_msg_fds(&msg);  in vhost_user_msg_handler()
2880 	if (msg.flags & VHOST_USER_NEED_REPLY) {  in vhost_user_msg_handler()
2881 		msg.payload.u64 = ret == RTE_VHOST_MSG_RESULT_ERR;  in vhost_user_msg_handler()
2882 		msg.size = sizeof(msg.payload.u64);  in vhost_user_msg_handler()
2883 		msg.fd_num = 0;  in vhost_user_msg_handler()
2884 		send_vhost_reply(fd, &msg);  in vhost_user_msg_handler()
2933 				       const struct VhostUserMsg *msg)  in process_slave_message_reply()  argument
2938 	if ((msg->flags & VHOST_USER_NEED_REPLY) == 0)  in process_slave_message_reply()
2954 	if (msg_reply.request.slave != msg->request.slave) {  in process_slave_message_reply()
2957 			msg_reply.request.slave, msg->request.slave);  in process_slave_message_reply()
2973 	struct VhostUserMsg msg = {  in vhost_user_iotlb_miss()  local
2976 		.size = sizeof(msg.payload.iotlb),  in vhost_user_iotlb_miss()
2984 	ret = send_vhost_message(dev->slave_req_fd, &msg);  in vhost_user_iotlb_miss()
2999 	struct VhostUserMsg msg = {  in vhost_user_slave_config_change()  local
3006 		msg.flags |= VHOST_USER_NEED_REPLY;  in vhost_user_slave_config_change()
3008 	ret = send_vhost_slave_message(dev, &msg);  in vhost_user_slave_config_change()
3016 	return process_slave_message_reply(dev, &msg);  in vhost_user_slave_config_change()
3037 	struct VhostUserMsg msg = {  in vhost_user_slave_set_vring_host_notifier()  local
3040 		.size = sizeof(msg.payload.area),  in vhost_user_slave_set_vring_host_notifier()
3049 		msg.payload.area.u64 |= VHOST_USER_VRING_NOFD_MASK;  in vhost_user_slave_set_vring_host_notifier()
3051 		msg.fds[0] = fd;  in vhost_user_slave_set_vring_host_notifier()
3052 		msg.fd_num = 1;  in vhost_user_slave_set_vring_host_notifier()
3055 	ret = send_vhost_slave_message(dev, &msg);  in vhost_user_slave_set_vring_host_notifier()
3062 	return process_slave_message_reply(dev, &msg);  in vhost_user_slave_set_vring_host_notifier()