26 vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq);
29 vhost_user_iotlb_pending_remove_all(struct vhost_virtqueue *vq)  in vhost_user_iotlb_pending_remove_all()  argument
33 	rte_rwlock_write_lock(&vq->iotlb_pending_lock);  in vhost_user_iotlb_pending_remove_all()
35 	RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next, temp_node) {  in vhost_user_iotlb_pending_remove_all()
36 		TAILQ_REMOVE(&vq->iotlb_pending_list, node, next);  in vhost_user_iotlb_pending_remove_all()
37 		rte_mempool_put(vq->iotlb_pool, node);  in vhost_user_iotlb_pending_remove_all()
40 	rte_rwlock_write_unlock(&vq->iotlb_pending_lock);  in vhost_user_iotlb_pending_remove_all()
44 vhost_user_iotlb_pending_miss(struct vhost_virtqueue *vq, uint64_t iova,  in vhost_user_iotlb_pending_miss()  argument
50 	rte_rwlock_read_lock(&vq->iotlb_pending_lock);  in vhost_user_iotlb_pending_miss()
52 	TAILQ_FOREACH(node, &vq->iotlb_pending_list, next) {  in vhost_user_iotlb_pending_miss()
59 	rte_rwlock_read_unlock(&vq->iotlb_pending_lock);  in vhost_user_iotlb_pending_miss()
65 vhost_user_iotlb_pending_insert(struct virtio_net *dev, struct vhost_virtqueue *vq,  in vhost_user_iotlb_pending_insert()  argument
71 	ret = rte_mempool_get(vq->iotlb_pool, (void **)&node);  in vhost_user_iotlb_pending_insert()
75 				dev->ifname, vq->iotlb_pool->name);  in vhost_user_iotlb_pending_insert()
76 		if (!TAILQ_EMPTY(&vq->iotlb_pending_list))  in vhost_user_iotlb_pending_insert()
77 			vhost_user_iotlb_pending_remove_all(vq);  in vhost_user_iotlb_pending_insert()
79 			vhost_user_iotlb_cache_random_evict(vq);  in vhost_user_iotlb_pending_insert()
80 		ret = rte_mempool_get(vq->iotlb_pool, (void **)&node);  in vhost_user_iotlb_pending_insert()
84 					dev->ifname, vq->iotlb_pool->name);  in vhost_user_iotlb_pending_insert()
92 	rte_rwlock_write_lock(&vq->iotlb_pending_lock);  in vhost_user_iotlb_pending_insert()
94 	TAILQ_INSERT_TAIL(&vq->iotlb_pending_list, node, next);  in vhost_user_iotlb_pending_insert()
96 	rte_rwlock_write_unlock(&vq->iotlb_pending_lock);  in vhost_user_iotlb_pending_insert()
100 vhost_user_iotlb_pending_remove(struct vhost_virtqueue *vq,  in vhost_user_iotlb_pending_remove()  argument
105 	rte_rwlock_write_lock(&vq->iotlb_pending_lock);  in vhost_user_iotlb_pending_remove()
107 	RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next,  in vhost_user_iotlb_pending_remove()
115 		TAILQ_REMOVE(&vq->iotlb_pending_list, node, next);  in vhost_user_iotlb_pending_remove()
116 		rte_mempool_put(vq->iotlb_pool, node);  in vhost_user_iotlb_pending_remove()
119 	rte_rwlock_write_unlock(&vq->iotlb_pending_lock);  in vhost_user_iotlb_pending_remove()
123 vhost_user_iotlb_cache_remove_all(struct vhost_virtqueue *vq)  in vhost_user_iotlb_cache_remove_all()  argument
127 	rte_rwlock_write_lock(&vq->iotlb_lock);  in vhost_user_iotlb_cache_remove_all()
129 	RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {  in vhost_user_iotlb_cache_remove_all()
130 		TAILQ_REMOVE(&vq->iotlb_list, node, next);  in vhost_user_iotlb_cache_remove_all()
131 		rte_mempool_put(vq->iotlb_pool, node);  in vhost_user_iotlb_cache_remove_all()
134 	vq->iotlb_cache_nr = 0;  in vhost_user_iotlb_cache_remove_all()
136 	rte_rwlock_write_unlock(&vq->iotlb_lock);  in vhost_user_iotlb_cache_remove_all()
140 vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq)  in vhost_user_iotlb_cache_random_evict()  argument
145 	rte_rwlock_write_lock(&vq->iotlb_lock);  in vhost_user_iotlb_cache_random_evict()
147 	entry_idx = rte_rand() % vq->iotlb_cache_nr;  in vhost_user_iotlb_cache_random_evict()
149 	RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {  in vhost_user_iotlb_cache_random_evict()
151 			TAILQ_REMOVE(&vq->iotlb_list, node, next);  in vhost_user_iotlb_cache_random_evict()
152 			rte_mempool_put(vq->iotlb_pool, node);  in vhost_user_iotlb_cache_random_evict()
153 			vq->iotlb_cache_nr--;  in vhost_user_iotlb_cache_random_evict()
159 	rte_rwlock_write_unlock(&vq->iotlb_lock);  in vhost_user_iotlb_cache_random_evict()
163 vhost_user_iotlb_cache_insert(struct virtio_net *dev, struct vhost_virtqueue *vq,  in vhost_user_iotlb_cache_insert()  argument
170 	ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node);  in vhost_user_iotlb_cache_insert()
174 				dev->ifname, vq->iotlb_pool->name);  in vhost_user_iotlb_cache_insert()
175 		if (!TAILQ_EMPTY(&vq->iotlb_list))  in vhost_user_iotlb_cache_insert()
176 			vhost_user_iotlb_cache_random_evict(vq);  in vhost_user_iotlb_cache_insert()
178 			vhost_user_iotlb_pending_remove_all(vq);  in vhost_user_iotlb_cache_insert()
179 		ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node);  in vhost_user_iotlb_cache_insert()
183 					dev->ifname, vq->iotlb_pool->name);  in vhost_user_iotlb_cache_insert()
193 	rte_rwlock_write_lock(&vq->iotlb_lock);  in vhost_user_iotlb_cache_insert()
195 	TAILQ_FOREACH(node, &vq->iotlb_list, next) {  in vhost_user_iotlb_cache_insert()
201 			rte_mempool_put(vq->iotlb_pool, new_node);  in vhost_user_iotlb_cache_insert()
205 			vq->iotlb_cache_nr++;  in vhost_user_iotlb_cache_insert()
210 	TAILQ_INSERT_TAIL(&vq->iotlb_list, new_node, next);  in vhost_user_iotlb_cache_insert()
211 	vq->iotlb_cache_nr++;  in vhost_user_iotlb_cache_insert()
214 	vhost_user_iotlb_pending_remove(vq, iova, size, perm);  in vhost_user_iotlb_cache_insert()
216 	rte_rwlock_write_unlock(&vq->iotlb_lock);  in vhost_user_iotlb_cache_insert()
221 vhost_user_iotlb_cache_remove(struct vhost_virtqueue *vq,  in vhost_user_iotlb_cache_remove()  argument
229 	rte_rwlock_write_lock(&vq->iotlb_lock);  in vhost_user_iotlb_cache_remove()
231 	RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {  in vhost_user_iotlb_cache_remove()
237 			TAILQ_REMOVE(&vq->iotlb_list, node, next);  in vhost_user_iotlb_cache_remove()
238 			rte_mempool_put(vq->iotlb_pool, node);  in vhost_user_iotlb_cache_remove()
239 			vq->iotlb_cache_nr--;  in vhost_user_iotlb_cache_remove()
243 	rte_rwlock_write_unlock(&vq->iotlb_lock);  in vhost_user_iotlb_cache_remove()
247 vhost_user_iotlb_cache_find(struct vhost_virtqueue *vq, uint64_t iova,  in vhost_user_iotlb_cache_find()  argument
256 	TAILQ_FOREACH(node, &vq->iotlb_list, next) {  in vhost_user_iotlb_cache_find()
289 vhost_user_iotlb_flush_all(struct vhost_virtqueue *vq)  in vhost_user_iotlb_flush_all()  argument
291 	vhost_user_iotlb_cache_remove_all(vq);  in vhost_user_iotlb_flush_all()
292 	vhost_user_iotlb_pending_remove_all(vq);  in vhost_user_iotlb_flush_all()
299 	struct vhost_virtqueue *vq = dev->virtqueue[vq_index];  in vhost_user_iotlb_init()  local
302 	if (vq->iotlb_pool) {  in vhost_user_iotlb_init()
307 		vhost_user_iotlb_flush_all(vq);  in vhost_user_iotlb_init()
311 	if (get_mempolicy(&socket, NULL, 0, vq, MPOL_F_NODE | MPOL_F_ADDR) != 0)  in vhost_user_iotlb_init()
315 	rte_rwlock_init(&vq->iotlb_lock);  in vhost_user_iotlb_init()
316 	rte_rwlock_init(&vq->iotlb_pending_lock);  in vhost_user_iotlb_init()
318 	TAILQ_INIT(&vq->iotlb_list);  in vhost_user_iotlb_init()
319 	TAILQ_INIT(&vq->iotlb_pending_list);  in vhost_user_iotlb_init()
326 	vq->iotlb_pool = rte_mempool_lookup(pool_name);  in vhost_user_iotlb_init()
327 	rte_mempool_free(vq->iotlb_pool);  in vhost_user_iotlb_init()
329 	vq->iotlb_pool = rte_mempool_create(pool_name,  in vhost_user_iotlb_init()
334 	if (!vq->iotlb_pool) {  in vhost_user_iotlb_init()
340 	vq->iotlb_cache_nr = 0;  in vhost_user_iotlb_init()