1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2017 Red Hat, Inc.
3 */
4
5 #ifdef RTE_LIBRTE_VHOST_NUMA
6 #include <numaif.h>
7 #endif
8
9 #include <rte_tailq.h>
10
11 #include "iotlb.h"
12 #include "vhost.h"
13
14 struct vhost_iotlb_entry {
15 TAILQ_ENTRY(vhost_iotlb_entry) next;
16
17 uint64_t iova;
18 uint64_t uaddr;
19 uint64_t size;
20 uint8_t perm;
21 };
22
23 #define IOTLB_CACHE_SIZE 2048
24
25 static void
26 vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq);
27
28 static void
vhost_user_iotlb_pending_remove_all(struct vhost_virtqueue * vq)29 vhost_user_iotlb_pending_remove_all(struct vhost_virtqueue *vq)
30 {
31 struct vhost_iotlb_entry *node, *temp_node;
32
33 rte_rwlock_write_lock(&vq->iotlb_pending_lock);
34
35 TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next, temp_node) {
36 TAILQ_REMOVE(&vq->iotlb_pending_list, node, next);
37 rte_mempool_put(vq->iotlb_pool, node);
38 }
39
40 rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
41 }
42
43 bool
vhost_user_iotlb_pending_miss(struct vhost_virtqueue * vq,uint64_t iova,uint8_t perm)44 vhost_user_iotlb_pending_miss(struct vhost_virtqueue *vq, uint64_t iova,
45 uint8_t perm)
46 {
47 struct vhost_iotlb_entry *node;
48 bool found = false;
49
50 rte_rwlock_read_lock(&vq->iotlb_pending_lock);
51
52 TAILQ_FOREACH(node, &vq->iotlb_pending_list, next) {
53 if ((node->iova == iova) && (node->perm == perm)) {
54 found = true;
55 break;
56 }
57 }
58
59 rte_rwlock_read_unlock(&vq->iotlb_pending_lock);
60
61 return found;
62 }
63
64 void
vhost_user_iotlb_pending_insert(struct vhost_virtqueue * vq,uint64_t iova,uint8_t perm)65 vhost_user_iotlb_pending_insert(struct vhost_virtqueue *vq,
66 uint64_t iova, uint8_t perm)
67 {
68 struct vhost_iotlb_entry *node;
69 int ret;
70
71 ret = rte_mempool_get(vq->iotlb_pool, (void **)&node);
72 if (ret) {
73 VHOST_LOG_CONFIG(DEBUG, "IOTLB pool empty, clear entries\n");
74 if (!TAILQ_EMPTY(&vq->iotlb_pending_list))
75 vhost_user_iotlb_pending_remove_all(vq);
76 else
77 vhost_user_iotlb_cache_random_evict(vq);
78 ret = rte_mempool_get(vq->iotlb_pool, (void **)&node);
79 if (ret) {
80 VHOST_LOG_CONFIG(ERR, "IOTLB pool still empty, failure\n");
81 return;
82 }
83 }
84
85 node->iova = iova;
86 node->perm = perm;
87
88 rte_rwlock_write_lock(&vq->iotlb_pending_lock);
89
90 TAILQ_INSERT_TAIL(&vq->iotlb_pending_list, node, next);
91
92 rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
93 }
94
95 void
vhost_user_iotlb_pending_remove(struct vhost_virtqueue * vq,uint64_t iova,uint64_t size,uint8_t perm)96 vhost_user_iotlb_pending_remove(struct vhost_virtqueue *vq,
97 uint64_t iova, uint64_t size, uint8_t perm)
98 {
99 struct vhost_iotlb_entry *node, *temp_node;
100
101 rte_rwlock_write_lock(&vq->iotlb_pending_lock);
102
103 TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next, temp_node) {
104 if (node->iova < iova)
105 continue;
106 if (node->iova >= iova + size)
107 continue;
108 if ((node->perm & perm) != node->perm)
109 continue;
110 TAILQ_REMOVE(&vq->iotlb_pending_list, node, next);
111 rte_mempool_put(vq->iotlb_pool, node);
112 }
113
114 rte_rwlock_write_unlock(&vq->iotlb_pending_lock);
115 }
116
117 static void
vhost_user_iotlb_cache_remove_all(struct vhost_virtqueue * vq)118 vhost_user_iotlb_cache_remove_all(struct vhost_virtqueue *vq)
119 {
120 struct vhost_iotlb_entry *node, *temp_node;
121
122 rte_rwlock_write_lock(&vq->iotlb_lock);
123
124 TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
125 TAILQ_REMOVE(&vq->iotlb_list, node, next);
126 rte_mempool_put(vq->iotlb_pool, node);
127 }
128
129 vq->iotlb_cache_nr = 0;
130
131 rte_rwlock_write_unlock(&vq->iotlb_lock);
132 }
133
134 static void
vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue * vq)135 vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq)
136 {
137 struct vhost_iotlb_entry *node, *temp_node;
138 int entry_idx;
139
140 rte_rwlock_write_lock(&vq->iotlb_lock);
141
142 entry_idx = rte_rand() % vq->iotlb_cache_nr;
143
144 TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
145 if (!entry_idx) {
146 TAILQ_REMOVE(&vq->iotlb_list, node, next);
147 rte_mempool_put(vq->iotlb_pool, node);
148 vq->iotlb_cache_nr--;
149 break;
150 }
151 entry_idx--;
152 }
153
154 rte_rwlock_write_unlock(&vq->iotlb_lock);
155 }
156
157 void
vhost_user_iotlb_cache_insert(struct vhost_virtqueue * vq,uint64_t iova,uint64_t uaddr,uint64_t size,uint8_t perm)158 vhost_user_iotlb_cache_insert(struct vhost_virtqueue *vq, uint64_t iova,
159 uint64_t uaddr, uint64_t size, uint8_t perm)
160 {
161 struct vhost_iotlb_entry *node, *new_node;
162 int ret;
163
164 ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node);
165 if (ret) {
166 VHOST_LOG_CONFIG(DEBUG, "IOTLB pool empty, clear entries\n");
167 if (!TAILQ_EMPTY(&vq->iotlb_list))
168 vhost_user_iotlb_cache_random_evict(vq);
169 else
170 vhost_user_iotlb_pending_remove_all(vq);
171 ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node);
172 if (ret) {
173 VHOST_LOG_CONFIG(ERR, "IOTLB pool still empty, failure\n");
174 return;
175 }
176 }
177
178 new_node->iova = iova;
179 new_node->uaddr = uaddr;
180 new_node->size = size;
181 new_node->perm = perm;
182
183 rte_rwlock_write_lock(&vq->iotlb_lock);
184
185 TAILQ_FOREACH(node, &vq->iotlb_list, next) {
186 /*
187 * Entries must be invalidated before being updated.
188 * So if iova already in list, assume identical.
189 */
190 if (node->iova == new_node->iova) {
191 rte_mempool_put(vq->iotlb_pool, new_node);
192 goto unlock;
193 } else if (node->iova > new_node->iova) {
194 TAILQ_INSERT_BEFORE(node, new_node, next);
195 vq->iotlb_cache_nr++;
196 goto unlock;
197 }
198 }
199
200 TAILQ_INSERT_TAIL(&vq->iotlb_list, new_node, next);
201 vq->iotlb_cache_nr++;
202
203 unlock:
204 vhost_user_iotlb_pending_remove(vq, iova, size, perm);
205
206 rte_rwlock_write_unlock(&vq->iotlb_lock);
207
208 }
209
210 void
vhost_user_iotlb_cache_remove(struct vhost_virtqueue * vq,uint64_t iova,uint64_t size)211 vhost_user_iotlb_cache_remove(struct vhost_virtqueue *vq,
212 uint64_t iova, uint64_t size)
213 {
214 struct vhost_iotlb_entry *node, *temp_node;
215
216 if (unlikely(!size))
217 return;
218
219 rte_rwlock_write_lock(&vq->iotlb_lock);
220
221 TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) {
222 /* Sorted list */
223 if (unlikely(iova + size < node->iova))
224 break;
225
226 if (iova < node->iova + node->size) {
227 TAILQ_REMOVE(&vq->iotlb_list, node, next);
228 rte_mempool_put(vq->iotlb_pool, node);
229 vq->iotlb_cache_nr--;
230 }
231 }
232
233 rte_rwlock_write_unlock(&vq->iotlb_lock);
234 }
235
236 uint64_t
vhost_user_iotlb_cache_find(struct vhost_virtqueue * vq,uint64_t iova,uint64_t * size,uint8_t perm)237 vhost_user_iotlb_cache_find(struct vhost_virtqueue *vq, uint64_t iova,
238 uint64_t *size, uint8_t perm)
239 {
240 struct vhost_iotlb_entry *node;
241 uint64_t offset, vva = 0, mapped = 0;
242
243 if (unlikely(!*size))
244 goto out;
245
246 TAILQ_FOREACH(node, &vq->iotlb_list, next) {
247 /* List sorted by iova */
248 if (unlikely(iova < node->iova))
249 break;
250
251 if (iova >= node->iova + node->size)
252 continue;
253
254 if (unlikely((perm & node->perm) != perm)) {
255 vva = 0;
256 break;
257 }
258
259 offset = iova - node->iova;
260 if (!vva)
261 vva = node->uaddr + offset;
262
263 mapped += node->size - offset;
264 iova = node->iova + node->size;
265
266 if (mapped >= *size)
267 break;
268 }
269
270 out:
271 /* Only part of the requested chunk is mapped */
272 if (unlikely(mapped < *size))
273 *size = mapped;
274
275 return vva;
276 }
277
278 void
vhost_user_iotlb_flush_all(struct vhost_virtqueue * vq)279 vhost_user_iotlb_flush_all(struct vhost_virtqueue *vq)
280 {
281 vhost_user_iotlb_cache_remove_all(vq);
282 vhost_user_iotlb_pending_remove_all(vq);
283 }
284
285 int
vhost_user_iotlb_init(struct virtio_net * dev,int vq_index)286 vhost_user_iotlb_init(struct virtio_net *dev, int vq_index)
287 {
288 char pool_name[RTE_MEMPOOL_NAMESIZE];
289 struct vhost_virtqueue *vq = dev->virtqueue[vq_index];
290 int socket = 0;
291
292 if (vq->iotlb_pool) {
293 /*
294 * The cache has already been initialized,
295 * just drop all cached and pending entries.
296 */
297 vhost_user_iotlb_flush_all(vq);
298 }
299
300 #ifdef RTE_LIBRTE_VHOST_NUMA
301 if (get_mempolicy(&socket, NULL, 0, vq, MPOL_F_NODE | MPOL_F_ADDR) != 0)
302 socket = 0;
303 #endif
304
305 rte_rwlock_init(&vq->iotlb_lock);
306 rte_rwlock_init(&vq->iotlb_pending_lock);
307
308 TAILQ_INIT(&vq->iotlb_list);
309 TAILQ_INIT(&vq->iotlb_pending_list);
310
311 snprintf(pool_name, sizeof(pool_name), "iotlb_%u_%d_%d",
312 getpid(), dev->vid, vq_index);
313 VHOST_LOG_CONFIG(DEBUG, "IOTLB cache name: %s\n", pool_name);
314
315 /* If already created, free it and recreate */
316 vq->iotlb_pool = rte_mempool_lookup(pool_name);
317 if (vq->iotlb_pool)
318 rte_mempool_free(vq->iotlb_pool);
319
320 vq->iotlb_pool = rte_mempool_create(pool_name,
321 IOTLB_CACHE_SIZE, sizeof(struct vhost_iotlb_entry), 0,
322 0, 0, NULL, NULL, NULL, socket,
323 MEMPOOL_F_NO_CACHE_ALIGN |
324 MEMPOOL_F_SP_PUT);
325 if (!vq->iotlb_pool) {
326 VHOST_LOG_CONFIG(ERR,
327 "Failed to create IOTLB cache pool (%s)\n",
328 pool_name);
329 return -1;
330 }
331
332 vq->iotlb_cache_nr = 0;
333
334 return 0;
335 }
336