106 	struct virtproc_info *vrp;  member
152 static struct rpmsg_device *__rpmsg_create_channel(struct virtproc_info *vrp,
209 static struct rpmsg_endpoint *__rpmsg_create_ept(struct virtproc_info *vrp,  in __rpmsg_create_ept()  argument
216 	struct device *dev = rpdev ? &rpdev->dev : &vrp->vdev->dev;  in __rpmsg_create_ept()
239 	mutex_lock(&vrp->endpoints_lock);  in __rpmsg_create_ept()
242 	id = idr_alloc(&vrp->endpoints, ept, id_min, id_max, GFP_KERNEL);  in __rpmsg_create_ept()
249 	mutex_unlock(&vrp->endpoints_lock);  in __rpmsg_create_ept()
254 	mutex_unlock(&vrp->endpoints_lock);  in __rpmsg_create_ept()
263 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_create_channel()  local
265 	return __rpmsg_create_channel(vrp, chinfo);  in virtio_rpmsg_create_channel()
272 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_release_channel()  local
274 	return rpmsg_unregister_device(&vrp->vdev->dev, chinfo);  in virtio_rpmsg_release_channel()
284 	return __rpmsg_create_ept(vch->vrp, rpdev, cb, priv, chinfo.src);  in virtio_rpmsg_create_ept()
298 __rpmsg_destroy_ept(struct virtproc_info *vrp, struct rpmsg_endpoint *ept)  in __rpmsg_destroy_ept()  argument
301 	mutex_lock(&vrp->endpoints_lock);  in __rpmsg_destroy_ept()
302 	idr_remove(&vrp->endpoints, ept->addr);  in __rpmsg_destroy_ept()
303 	mutex_unlock(&vrp->endpoints_lock);  in __rpmsg_destroy_ept()
317 	__rpmsg_destroy_ept(vch->vrp, ept);  in virtio_rpmsg_destroy_ept()
323 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_announce_create()  local
329 	    virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) {  in virtio_rpmsg_announce_create()
347 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_announce_destroy()  local
353 	    virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) {  in virtio_rpmsg_announce_destroy()
390 static struct rpmsg_device *__rpmsg_create_channel(struct virtproc_info *vrp,  in __rpmsg_create_channel()  argument
395 	struct device *tmp, *dev = &vrp->vdev->dev;  in __rpmsg_create_channel()
413 	vch->vrp = vrp;  in __rpmsg_create_channel()
420 	rpdev->little_endian = virtio_is_little_endian(vrp->vdev);  in __rpmsg_create_channel()
430 	rpdev->dev.parent = &vrp->vdev->dev;  in __rpmsg_create_channel()
440 static void *get_a_tx_buf(struct virtproc_info *vrp)  in get_a_tx_buf()  argument
446 	mutex_lock(&vrp->tx_lock);  in get_a_tx_buf()
452 	if (vrp->last_sbuf < vrp->num_bufs / 2)  in get_a_tx_buf()
453 		ret = vrp->sbufs + vrp->buf_size * vrp->last_sbuf++;  in get_a_tx_buf()
456 		ret = virtqueue_get_buf(vrp->svq, &len);  in get_a_tx_buf()
458 	mutex_unlock(&vrp->tx_lock);  in get_a_tx_buf()
479 static void rpmsg_upref_sleepers(struct virtproc_info *vrp)  in rpmsg_upref_sleepers()  argument
482 	mutex_lock(&vrp->tx_lock);  in rpmsg_upref_sleepers()
485 	if (atomic_inc_return(&vrp->sleepers) == 1)  in rpmsg_upref_sleepers()
487 		virtqueue_enable_cb(vrp->svq);  in rpmsg_upref_sleepers()
489 	mutex_unlock(&vrp->tx_lock);  in rpmsg_upref_sleepers()
506 static void rpmsg_downref_sleepers(struct virtproc_info *vrp)  in rpmsg_downref_sleepers()  argument
509 	mutex_lock(&vrp->tx_lock);  in rpmsg_downref_sleepers()
512 	if (atomic_dec_and_test(&vrp->sleepers))  in rpmsg_downref_sleepers()
514 		virtqueue_disable_cb(vrp->svq);  in rpmsg_downref_sleepers()
516 	mutex_unlock(&vrp->tx_lock);  in rpmsg_downref_sleepers()
558 	struct virtproc_info *vrp = vch->vrp;  in rpmsg_send_offchannel_raw()  local
579 	if (len > vrp->buf_size - sizeof(struct rpmsg_hdr)) {  in rpmsg_send_offchannel_raw()
585 	msg = get_a_tx_buf(vrp);  in rpmsg_send_offchannel_raw()
592 		rpmsg_upref_sleepers(vrp);  in rpmsg_send_offchannel_raw()
600 		err = wait_event_interruptible_timeout(vrp->sendq,  in rpmsg_send_offchannel_raw()
601 					(msg = get_a_tx_buf(vrp)),  in rpmsg_send_offchannel_raw()
605 		rpmsg_downref_sleepers(vrp);  in rpmsg_send_offchannel_raw()
630 	mutex_lock(&vrp->tx_lock);  in rpmsg_send_offchannel_raw()
633 	err = virtqueue_add_outbuf(vrp->svq, &sg, 1, msg, GFP_KERNEL);  in rpmsg_send_offchannel_raw()
645 	virtqueue_kick(vrp->svq);  in rpmsg_send_offchannel_raw()
647 	mutex_unlock(&vrp->tx_lock);  in rpmsg_send_offchannel_raw()
706 	return vch->vrp->buf_size - sizeof(struct rpmsg_hdr);  in virtio_rpmsg_get_mtu()
709 static int rpmsg_recv_single(struct virtproc_info *vrp, struct device *dev,  in rpmsg_recv_single()  argument
714 	bool little_endian = virtio_is_little_endian(vrp->vdev);  in rpmsg_recv_single()
732 	if (len > vrp->buf_size ||  in rpmsg_recv_single()
739 	mutex_lock(&vrp->endpoints_lock);  in rpmsg_recv_single()
741 	ept = idr_find(&vrp->endpoints, __rpmsg32_to_cpu(little_endian, msg->dst));  in rpmsg_recv_single()
747 	mutex_unlock(&vrp->endpoints_lock);  in rpmsg_recv_single()
765 	rpmsg_sg_init(&sg, msg, vrp->buf_size);  in rpmsg_recv_single()
768 	err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, msg, GFP_KERNEL);  in rpmsg_recv_single()
780 	struct virtproc_info *vrp = rvq->vdev->priv;  in rpmsg_recv_done()  local
793 		err = rpmsg_recv_single(vrp, dev, msg, len);  in rpmsg_recv_done()
806 		virtqueue_kick(vrp->rvq);  in rpmsg_recv_done()
818 	struct virtproc_info *vrp = svq->vdev->priv;  in rpmsg_xmit_done()  local
823 	wake_up_interruptible(&vrp->sendq);  in rpmsg_xmit_done()
833 	struct virtproc_info *vrp = vdev->priv;  in rpmsg_virtio_add_ctrl_dev()  local
843 	vch->vrp = vrp;  in rpmsg_virtio_add_ctrl_dev()
849 	rpdev_ctrl->dev.parent = &vrp->vdev->dev;  in rpmsg_virtio_add_ctrl_dev()
851 	rpdev_ctrl->little_endian = virtio_is_little_endian(vrp->vdev);  in rpmsg_virtio_add_ctrl_dev()
876 	struct virtproc_info *vrp;  in rpmsg_probe()  local
884 	vrp = kzalloc(sizeof(*vrp), GFP_KERNEL);  in rpmsg_probe()
885 	if (!vrp)  in rpmsg_probe()
888 	vrp->vdev = vdev;  in rpmsg_probe()
890 	idr_init(&vrp->endpoints);  in rpmsg_probe()
891 	mutex_init(&vrp->endpoints_lock);  in rpmsg_probe()
892 	mutex_init(&vrp->tx_lock);  in rpmsg_probe()
893 	init_waitqueue_head(&vrp->sendq);  in rpmsg_probe()
900 	vrp->rvq = vqs[0];  in rpmsg_probe()
901 	vrp->svq = vqs[1];  in rpmsg_probe()
904 	WARN_ON(virtqueue_get_vring_size(vrp->rvq) !=  in rpmsg_probe()
905 		virtqueue_get_vring_size(vrp->svq));  in rpmsg_probe()
908 	if (virtqueue_get_vring_size(vrp->rvq) < MAX_RPMSG_NUM_BUFS / 2)  in rpmsg_probe()
909 		vrp->num_bufs = virtqueue_get_vring_size(vrp->rvq) * 2;  in rpmsg_probe()
911 		vrp->num_bufs = MAX_RPMSG_NUM_BUFS;  in rpmsg_probe()
913 	vrp->buf_size = MAX_RPMSG_BUF_SIZE;  in rpmsg_probe()
915 	total_buf_space = vrp->num_bufs * vrp->buf_size;  in rpmsg_probe()
919 				     total_buf_space, &vrp->bufs_dma,  in rpmsg_probe()
927 		bufs_va, &vrp->bufs_dma);  in rpmsg_probe()
930 	vrp->rbufs = bufs_va;  in rpmsg_probe()
933 	vrp->sbufs = bufs_va + total_buf_space / 2;  in rpmsg_probe()
936 	for (i = 0; i < vrp->num_bufs / 2; i++) {  in rpmsg_probe()
938 		void *cpu_addr = vrp->rbufs + i * vrp->buf_size;  in rpmsg_probe()
940 		rpmsg_sg_init(&sg, cpu_addr, vrp->buf_size);  in rpmsg_probe()
942 		err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, cpu_addr,  in rpmsg_probe()
948 	virtqueue_disable_cb(vrp->svq);  in rpmsg_probe()
950 	vdev->priv = vrp;  in rpmsg_probe()
967 		vch->vrp = vrp;  in rpmsg_probe()
972 		rpdev_ns->little_endian = virtio_is_little_endian(vrp->vdev);  in rpmsg_probe()
974 		rpdev_ns->dev.parent = &vrp->vdev->dev;  in rpmsg_probe()
987 	notify = virtqueue_kick_prepare(vrp->rvq);  in rpmsg_probe()
998 		virtqueue_notify(vrp->rvq);  in rpmsg_probe()
1008 			  bufs_va, vrp->bufs_dma);  in rpmsg_probe()
1010 	vdev->config->del_vqs(vrp->vdev);  in rpmsg_probe()
1012 	kfree(vrp);  in rpmsg_probe()
1025 	struct virtproc_info *vrp = vdev->priv;  in rpmsg_remove()  local
1026 	size_t total_buf_space = vrp->num_bufs * vrp->buf_size;  in rpmsg_remove()
1035 	idr_destroy(&vrp->endpoints);  in rpmsg_remove()
1037 	vdev->config->del_vqs(vrp->vdev);  in rpmsg_remove()
1040 			  vrp->rbufs, vrp->bufs_dma);  in rpmsg_remove()
1042 	kfree(vrp);  in rpmsg_remove()