1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2019 Mellanox Technologies, Ltd
3 */
4 #include <stdlib.h>
5
6 #include <rte_malloc.h>
7 #include <rte_errno.h>
8 #include <rte_common.h>
9 #include <rte_sched_common.h>
10
11 #include <mlx5_prm.h>
12 #include <mlx5_common.h>
13
14 #include "mlx5_vdpa_utils.h"
15 #include "mlx5_vdpa.h"
16
17 void
mlx5_vdpa_mem_dereg(struct mlx5_vdpa_priv * priv)18 mlx5_vdpa_mem_dereg(struct mlx5_vdpa_priv *priv)
19 {
20 struct mlx5_vdpa_query_mr *entry;
21 struct mlx5_vdpa_query_mr *next;
22
23 entry = SLIST_FIRST(&priv->mr_list);
24 while (entry) {
25 next = SLIST_NEXT(entry, next);
26 if (entry->is_indirect)
27 claim_zero(mlx5_devx_cmd_destroy(entry->mkey));
28 else
29 claim_zero(mlx5_glue->dereg_mr(entry->mr));
30 SLIST_REMOVE(&priv->mr_list, entry, mlx5_vdpa_query_mr, next);
31 rte_free(entry);
32 entry = next;
33 }
34 SLIST_INIT(&priv->mr_list);
35 if (priv->vmem) {
36 free(priv->vmem);
37 priv->vmem = NULL;
38 }
39 }
40
41 static int
mlx5_vdpa_regions_addr_cmp(const void * a,const void * b)42 mlx5_vdpa_regions_addr_cmp(const void *a, const void *b)
43 {
44 const struct rte_vhost_mem_region *region_a = a;
45 const struct rte_vhost_mem_region *region_b = b;
46
47 if (region_a->guest_phys_addr < region_b->guest_phys_addr)
48 return -1;
49 if (region_a->guest_phys_addr > region_b->guest_phys_addr)
50 return 1;
51 return 0;
52 }
53
54 #define KLM_NUM_MAX_ALIGN(sz) (RTE_ALIGN_CEIL(sz, MLX5_MAX_KLM_BYTE_COUNT) / \
55 MLX5_MAX_KLM_BYTE_COUNT)
56
57 /*
58 * Allocate and sort the region list and choose indirect mkey mode:
59 * 1. Calculate GCD, guest memory size and indirect mkey entries num per mode.
60 * 2. Align GCD to the maximum allowed size(2G) and to be power of 2.
61 * 2. Decide the indirect mkey mode according to the next rules:
62 * a. If both KLM_FBS entries number and KLM entries number are bigger
63 * than the maximum allowed(MLX5_DEVX_MAX_KLM_ENTRIES) - error.
64 * b. KLM mode if KLM_FBS entries number is bigger than the maximum
65 * allowed(MLX5_DEVX_MAX_KLM_ENTRIES).
66 * c. KLM mode if GCD is smaller than the minimum allowed(4K).
67 * d. KLM mode if the total size of KLM entries is in one cache line
68 * and the total size of KLM_FBS entries is not in one cache line.
69 * e. Otherwise, KLM_FBS mode.
70 */
71 static struct rte_vhost_memory *
mlx5_vdpa_vhost_mem_regions_prepare(int vid,uint8_t * mode,uint64_t * mem_size,uint64_t * gcd,uint32_t * entries_num)72 mlx5_vdpa_vhost_mem_regions_prepare(int vid, uint8_t *mode, uint64_t *mem_size,
73 uint64_t *gcd, uint32_t *entries_num)
74 {
75 struct rte_vhost_memory *mem;
76 uint64_t size;
77 uint64_t klm_entries_num = 0;
78 uint64_t klm_fbs_entries_num;
79 uint32_t i;
80 int ret = rte_vhost_get_mem_table(vid, &mem);
81
82 if (ret < 0) {
83 DRV_LOG(ERR, "Failed to get VM memory layout vid =%d.", vid);
84 rte_errno = EINVAL;
85 return NULL;
86 }
87 qsort(mem->regions, mem->nregions, sizeof(mem->regions[0]),
88 mlx5_vdpa_regions_addr_cmp);
89 *mem_size = (mem->regions[(mem->nregions - 1)].guest_phys_addr) +
90 (mem->regions[(mem->nregions - 1)].size) -
91 (mem->regions[0].guest_phys_addr);
92 *gcd = 0;
93 for (i = 0; i < mem->nregions; ++i) {
94 DRV_LOG(INFO, "Region %u: HVA 0x%" PRIx64 ", GPA 0x%" PRIx64
95 ", size 0x%" PRIx64 ".", i,
96 mem->regions[i].host_user_addr,
97 mem->regions[i].guest_phys_addr, mem->regions[i].size);
98 if (i > 0) {
99 /* Hole handle. */
100 size = mem->regions[i].guest_phys_addr -
101 (mem->regions[i - 1].guest_phys_addr +
102 mem->regions[i - 1].size);
103 *gcd = rte_get_gcd64(*gcd, size);
104 klm_entries_num += KLM_NUM_MAX_ALIGN(size);
105 }
106 size = mem->regions[i].size;
107 *gcd = rte_get_gcd64(*gcd, size);
108 klm_entries_num += KLM_NUM_MAX_ALIGN(size);
109 }
110 if (*gcd > MLX5_MAX_KLM_BYTE_COUNT)
111 *gcd = rte_get_gcd64(*gcd, MLX5_MAX_KLM_BYTE_COUNT);
112 if (!RTE_IS_POWER_OF_2(*gcd)) {
113 uint64_t candidate_gcd = rte_align64prevpow2(*gcd);
114
115 while (candidate_gcd > 1 && (*gcd % candidate_gcd))
116 candidate_gcd /= 2;
117 DRV_LOG(DEBUG, "GCD 0x%" PRIx64 " is not power of 2. Adjusted "
118 "GCD is 0x%" PRIx64 ".", *gcd, candidate_gcd);
119 *gcd = candidate_gcd;
120 }
121 klm_fbs_entries_num = *mem_size / *gcd;
122 if (*gcd < MLX5_MIN_KLM_FIXED_BUFFER_SIZE || klm_fbs_entries_num >
123 MLX5_DEVX_MAX_KLM_ENTRIES ||
124 ((klm_entries_num * sizeof(struct mlx5_klm)) <=
125 RTE_CACHE_LINE_SIZE && (klm_fbs_entries_num *
126 sizeof(struct mlx5_klm)) >
127 RTE_CACHE_LINE_SIZE)) {
128 *mode = MLX5_MKC_ACCESS_MODE_KLM;
129 *entries_num = klm_entries_num;
130 DRV_LOG(INFO, "Indirect mkey mode is KLM.");
131 } else {
132 *mode = MLX5_MKC_ACCESS_MODE_KLM_FBS;
133 *entries_num = klm_fbs_entries_num;
134 DRV_LOG(INFO, "Indirect mkey mode is KLM Fixed Buffer Size.");
135 }
136 DRV_LOG(DEBUG, "Memory registration information: nregions = %u, "
137 "mem_size = 0x%" PRIx64 ", GCD = 0x%" PRIx64
138 ", klm_fbs_entries_num = 0x%" PRIx64 ", klm_entries_num = 0x%"
139 PRIx64 ".", mem->nregions, *mem_size, *gcd, klm_fbs_entries_num,
140 klm_entries_num);
141 if (*entries_num > MLX5_DEVX_MAX_KLM_ENTRIES) {
142 DRV_LOG(ERR, "Failed to prepare memory of vid %d - memory is "
143 "too fragmented.", vid);
144 free(mem);
145 return NULL;
146 }
147 return mem;
148 }
149
150 static int
mlx5_vdpa_mem_cmp(struct rte_vhost_memory * mem1,struct rte_vhost_memory * mem2)151 mlx5_vdpa_mem_cmp(struct rte_vhost_memory *mem1, struct rte_vhost_memory *mem2)
152 {
153 uint32_t i;
154
155 if (mem1->nregions != mem2->nregions)
156 return -1;
157 for (i = 0; i < mem1->nregions; i++) {
158 if (mem1->regions[i].guest_phys_addr !=
159 mem2->regions[i].guest_phys_addr)
160 return -1;
161 if (mem1->regions[i].size != mem2->regions[i].size)
162 return -1;
163 }
164 return 0;
165 }
166
167 #define KLM_SIZE_MAX_ALIGN(sz) ((sz) > MLX5_MAX_KLM_BYTE_COUNT ? \
168 MLX5_MAX_KLM_BYTE_COUNT : (sz))
169
170 /*
171 * The target here is to group all the physical memory regions of the
172 * virtio device in one indirect mkey.
173 * For KLM Fixed Buffer Size mode (HW find the translation entry in one
174 * read according to the guest physical address):
175 * All the sub-direct mkeys of it must be in the same size, hence, each
176 * one of them should be in the GCD size of all the virtio memory
177 * regions and the holes between them.
178 * For KLM mode (each entry may be in different size so HW must iterate
179 * the entries):
180 * Each virtio memory region and each hole between them have one entry,
181 * just need to cover the maximum allowed size(2G) by splitting entries
182 * which their associated memory regions are bigger than 2G.
183 * It means that each virtio memory region may be mapped to more than
184 * one direct mkey in the 2 modes.
185 * All the holes of invalid memory between the virtio memory regions
186 * will be mapped to the null memory region for security.
187 */
188 int
mlx5_vdpa_mem_register(struct mlx5_vdpa_priv * priv)189 mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv)
190 {
191 struct mlx5_devx_mkey_attr mkey_attr;
192 struct mlx5_vdpa_query_mr *entry = NULL;
193 struct rte_vhost_mem_region *reg = NULL;
194 uint8_t mode = 0;
195 uint32_t entries_num = 0;
196 uint32_t i;
197 uint64_t gcd = 0;
198 uint64_t klm_size;
199 uint64_t mem_size;
200 uint64_t k;
201 int klm_index = 0;
202 int ret;
203 struct rte_vhost_memory *mem = mlx5_vdpa_vhost_mem_regions_prepare
204 (priv->vid, &mode, &mem_size, &gcd, &entries_num);
205 struct mlx5_klm klm_array[entries_num];
206
207 if (!mem)
208 return -rte_errno;
209 if (priv->vmem != NULL) {
210 if (mlx5_vdpa_mem_cmp(mem, priv->vmem) == 0) {
211 /* VM memory not changed, reuse resources. */
212 free(mem);
213 return 0;
214 }
215 mlx5_vdpa_mem_dereg(priv);
216 }
217 priv->vmem = mem;
218 for (i = 0; i < mem->nregions; i++) {
219 reg = &mem->regions[i];
220 entry = rte_zmalloc(__func__, sizeof(*entry), 0);
221 if (!entry) {
222 ret = -ENOMEM;
223 DRV_LOG(ERR, "Failed to allocate mem entry memory.");
224 goto error;
225 }
226 entry->mr = mlx5_glue->reg_mr_iova(priv->cdev->pd,
227 (void *)(uintptr_t)(reg->host_user_addr),
228 reg->size, reg->guest_phys_addr,
229 IBV_ACCESS_LOCAL_WRITE);
230 if (!entry->mr) {
231 DRV_LOG(ERR, "Failed to create direct Mkey.");
232 ret = -rte_errno;
233 goto error;
234 }
235 entry->is_indirect = 0;
236 if (i > 0) {
237 uint64_t sadd;
238 uint64_t empty_region_sz = reg->guest_phys_addr -
239 (mem->regions[i - 1].guest_phys_addr +
240 mem->regions[i - 1].size);
241
242 if (empty_region_sz > 0) {
243 sadd = mem->regions[i - 1].guest_phys_addr +
244 mem->regions[i - 1].size;
245 klm_size = mode == MLX5_MKC_ACCESS_MODE_KLM ?
246 KLM_SIZE_MAX_ALIGN(empty_region_sz) : gcd;
247 for (k = 0; k < empty_region_sz;
248 k += klm_size) {
249 klm_array[klm_index].byte_count =
250 k + klm_size > empty_region_sz ?
251 empty_region_sz - k : klm_size;
252 klm_array[klm_index].mkey =
253 priv->null_mr->lkey;
254 klm_array[klm_index].address = sadd + k;
255 klm_index++;
256 }
257 }
258 }
259 klm_size = mode == MLX5_MKC_ACCESS_MODE_KLM ?
260 KLM_SIZE_MAX_ALIGN(reg->size) : gcd;
261 for (k = 0; k < reg->size; k += klm_size) {
262 klm_array[klm_index].byte_count = k + klm_size >
263 reg->size ? reg->size - k : klm_size;
264 klm_array[klm_index].mkey = entry->mr->lkey;
265 klm_array[klm_index].address = reg->guest_phys_addr + k;
266 klm_index++;
267 }
268 SLIST_INSERT_HEAD(&priv->mr_list, entry, next);
269 }
270 memset(&mkey_attr, 0, sizeof(mkey_attr));
271 mkey_attr.addr = (uintptr_t)(mem->regions[0].guest_phys_addr);
272 mkey_attr.size = mem_size;
273 mkey_attr.pd = priv->cdev->pdn;
274 mkey_attr.umem_id = 0;
275 /* Must be zero for KLM mode. */
276 mkey_attr.log_entity_size = mode == MLX5_MKC_ACCESS_MODE_KLM_FBS ?
277 rte_log2_u64(gcd) : 0;
278 mkey_attr.pg_access = 0;
279 mkey_attr.klm_array = klm_array;
280 mkey_attr.klm_num = klm_index;
281 entry = rte_zmalloc(__func__, sizeof(*entry), 0);
282 if (!entry) {
283 DRV_LOG(ERR, "Failed to allocate memory for indirect entry.");
284 ret = -ENOMEM;
285 goto error;
286 }
287 entry->mkey = mlx5_devx_cmd_mkey_create(priv->cdev->ctx, &mkey_attr);
288 if (!entry->mkey) {
289 DRV_LOG(ERR, "Failed to create indirect Mkey.");
290 ret = -rte_errno;
291 goto error;
292 }
293 entry->is_indirect = 1;
294 SLIST_INSERT_HEAD(&priv->mr_list, entry, next);
295 priv->gpa_mkey_index = entry->mkey->id;
296 return 0;
297 error:
298 rte_free(entry);
299 mlx5_vdpa_mem_dereg(priv);
300 rte_errno = -ret;
301 return ret;
302 }
303