1 /*- 2 * BSD LICENSE 3 * 4 * Copyright 2016 6WIND S.A. 5 * Copyright 2016 Mellanox. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of 6WIND S.A. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* Verbs header. */ 35 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 36 #ifdef PEDANTIC 37 #pragma GCC diagnostic ignored "-Wpedantic" 38 #endif 39 #include <infiniband/verbs.h> 40 #ifdef PEDANTIC 41 #pragma GCC diagnostic error "-Wpedantic" 42 #endif 43 44 #include <rte_mempool.h> 45 #include <rte_malloc.h> 46 47 #include "mlx5.h" 48 #include "mlx5_rxtx.h" 49 50 struct mlx5_check_mempool_data { 51 int ret; 52 char *start; 53 char *end; 54 }; 55 56 /* Called by mlx5_check_mempool() when iterating the memory chunks. */ 57 static void 58 mlx5_check_mempool_cb(struct rte_mempool *mp __rte_unused, 59 void *opaque, struct rte_mempool_memhdr *memhdr, 60 unsigned int mem_idx __rte_unused) 61 { 62 struct mlx5_check_mempool_data *data = opaque; 63 64 /* It already failed, skip the next chunks. */ 65 if (data->ret != 0) 66 return; 67 /* It is the first chunk. */ 68 if (data->start == NULL && data->end == NULL) { 69 data->start = memhdr->addr; 70 data->end = data->start + memhdr->len; 71 return; 72 } 73 if (data->end == memhdr->addr) { 74 data->end += memhdr->len; 75 return; 76 } 77 if (data->start == (char *)memhdr->addr + memhdr->len) { 78 data->start -= memhdr->len; 79 return; 80 } 81 /* Error, mempool is not virtually contiguous. */ 82 data->ret = -1; 83 } 84 85 /** 86 * Check if a mempool can be used: it must be virtually contiguous. 87 * 88 * @param[in] mp 89 * Pointer to memory pool. 90 * @param[out] start 91 * Pointer to the start address of the mempool virtual memory area 92 * @param[out] end 93 * Pointer to the end address of the mempool virtual memory area 94 * 95 * @return 96 * 0 on success (mempool is virtually contiguous), -1 on error. 97 */ 98 static int 99 mlx5_check_mempool(struct rte_mempool *mp, uintptr_t *start, 100 uintptr_t *end) 101 { 102 struct mlx5_check_mempool_data data; 103 104 memset(&data, 0, sizeof(data)); 105 rte_mempool_mem_iter(mp, mlx5_check_mempool_cb, &data); 106 *start = (uintptr_t)data.start; 107 *end = (uintptr_t)data.end; 108 return data.ret; 109 } 110 111 /** 112 * Register a Memory Region (MR) <-> Memory Pool (MP) association in 113 * txq->mp2mr[]. If mp2mr[] is full, remove an entry first. 114 * 115 * @param txq 116 * Pointer to TX queue structure. 117 * @param[in] mp 118 * Memory Pool for which a Memory Region lkey must be returned. 119 * @param idx 120 * Index of the next available entry. 121 * 122 * @return 123 * mr on success, NULL on failure and rte_errno is set. 124 */ 125 struct mlx5_mr * 126 mlx5_txq_mp2mr_reg(struct mlx5_txq_data *txq, struct rte_mempool *mp, 127 unsigned int idx) 128 { 129 struct mlx5_txq_ctrl *txq_ctrl = 130 container_of(txq, struct mlx5_txq_ctrl, txq); 131 struct rte_eth_dev *dev; 132 struct mlx5_mr *mr; 133 134 rte_spinlock_lock(&txq_ctrl->priv->mr_lock); 135 /* Add a new entry, register MR first. */ 136 DRV_LOG(DEBUG, "port %u discovered new memory pool \"%s\" (%p)", 137 PORT_ID(txq_ctrl->priv), mp->name, (void *)mp); 138 dev = ETH_DEV(txq_ctrl->priv); 139 mr = mlx5_mr_get(dev, mp); 140 if (mr == NULL) { 141 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 142 DRV_LOG(DEBUG, 143 "port %u using unregistered mempool 0x%p(%s)" 144 " in secondary process, please create mempool" 145 " before rte_eth_dev_start()", 146 PORT_ID(txq_ctrl->priv), (void *)mp, mp->name); 147 rte_spinlock_unlock(&txq_ctrl->priv->mr_lock); 148 rte_errno = ENOTSUP; 149 return NULL; 150 } 151 mr = mlx5_mr_new(dev, mp); 152 } 153 if (unlikely(mr == NULL)) { 154 DRV_LOG(DEBUG, 155 "port %u unable to configure memory region," 156 " ibv_reg_mr() failed.", 157 PORT_ID(txq_ctrl->priv)); 158 rte_spinlock_unlock(&txq_ctrl->priv->mr_lock); 159 return NULL; 160 } 161 if (unlikely(idx == RTE_DIM(txq->mp2mr))) { 162 /* Table is full, remove oldest entry. */ 163 DRV_LOG(DEBUG, 164 "port %u memory region <-> memory pool table full, " 165 " dropping oldest entry", 166 PORT_ID(txq_ctrl->priv)); 167 --idx; 168 mlx5_mr_release(txq->mp2mr[0]); 169 memmove(&txq->mp2mr[0], &txq->mp2mr[1], 170 (sizeof(txq->mp2mr) - sizeof(txq->mp2mr[0]))); 171 } 172 /* Store the new entry. */ 173 txq_ctrl->txq.mp2mr[idx] = mr; 174 DRV_LOG(DEBUG, 175 "port %u new memory region lkey for MP \"%s\" (%p): 0x%08" 176 PRIu32, 177 PORT_ID(txq_ctrl->priv), mp->name, (void *)mp, 178 txq_ctrl->txq.mp2mr[idx]->lkey); 179 rte_spinlock_unlock(&txq_ctrl->priv->mr_lock); 180 return mr; 181 } 182 183 struct mlx5_mp2mr_mbuf_check_data { 184 int ret; 185 }; 186 187 /** 188 * Callback function for rte_mempool_obj_iter() to check whether a given 189 * mempool object looks like a mbuf. 190 * 191 * @param[in] mp 192 * The mempool pointer 193 * @param[in] arg 194 * Context data (struct txq_mp2mr_mbuf_check_data). Contains the 195 * return value. 196 * @param[in] obj 197 * Object address. 198 * @param index 199 * Object index, unused. 200 */ 201 static void 202 txq_mp2mr_mbuf_check(struct rte_mempool *mp, void *arg, void *obj, 203 uint32_t index __rte_unused) 204 { 205 struct mlx5_mp2mr_mbuf_check_data *data = arg; 206 struct rte_mbuf *buf = obj; 207 208 /* 209 * Check whether mbuf structure fits element size and whether mempool 210 * pointer is valid. 211 */ 212 if (sizeof(*buf) > mp->elt_size || buf->pool != mp) 213 data->ret = -1; 214 } 215 216 /** 217 * Iterator function for rte_mempool_walk() to register existing mempools and 218 * fill the MP to MR cache of a TX queue. 219 * 220 * @param[in] mp 221 * Memory Pool to register. 222 * @param *arg 223 * Pointer to TX queue structure. 224 */ 225 void 226 mlx5_mp2mr_iter(struct rte_mempool *mp, void *arg) 227 { 228 struct priv *priv = (struct priv *)arg; 229 struct mlx5_mp2mr_mbuf_check_data data = { 230 .ret = 0, 231 }; 232 struct mlx5_mr *mr; 233 234 /* Register mempool only if the first element looks like a mbuf. */ 235 if (rte_mempool_obj_iter(mp, txq_mp2mr_mbuf_check, &data) == 0 || 236 data.ret == -1) 237 return; 238 mr = mlx5_mr_get(ETH_DEV(priv), mp); 239 if (mr) { 240 mlx5_mr_release(mr); 241 return; 242 } 243 mr = mlx5_mr_new(ETH_DEV(priv), mp); 244 if (!mr) 245 DRV_LOG(ERR, "port %u cannot create memory region: %s", 246 PORT_ID(priv), strerror(rte_errno)); 247 } 248 249 /** 250 * Register a new memory region from the mempool and store it in the memory 251 * region list. 252 * 253 * @param dev 254 * Pointer to Ethernet device. 255 * @param mp 256 * Pointer to the memory pool to register. 257 * 258 * @return 259 * The memory region on success, NULL on failure and rte_errno is set. 260 */ 261 struct mlx5_mr * 262 mlx5_mr_new(struct rte_eth_dev *dev, struct rte_mempool *mp) 263 { 264 struct priv *priv = dev->data->dev_private; 265 const struct rte_memseg *ms = rte_eal_get_physmem_layout(); 266 uintptr_t start; 267 uintptr_t end; 268 unsigned int i; 269 struct mlx5_mr *mr; 270 271 mr = rte_zmalloc_socket(__func__, sizeof(*mr), 0, mp->socket_id); 272 if (!mr) { 273 DRV_LOG(DEBUG, 274 "port %u unable to configure memory region," 275 " ibv_reg_mr() failed.", 276 dev->data->port_id); 277 rte_errno = ENOMEM; 278 return NULL; 279 } 280 if (mlx5_check_mempool(mp, &start, &end) != 0) { 281 DRV_LOG(ERR, "port %u mempool %p: not virtually contiguous", 282 dev->data->port_id, (void *)mp); 283 rte_errno = ENOMEM; 284 return NULL; 285 } 286 DRV_LOG(DEBUG, "port %u mempool %p area start=%p end=%p size=%zu", 287 dev->data->port_id, (void *)mp, (void *)start, (void *)end, 288 (size_t)(end - start)); 289 /* Save original addresses for exact MR lookup. */ 290 mr->start = start; 291 mr->end = end; 292 /* Round start and end to page boundary if found in memory segments. */ 293 for (i = 0; (i < RTE_MAX_MEMSEG) && (ms[i].addr != NULL); ++i) { 294 uintptr_t addr = (uintptr_t)ms[i].addr; 295 size_t len = ms[i].len; 296 unsigned int align = ms[i].hugepage_sz; 297 298 if ((start > addr) && (start < addr + len)) 299 start = RTE_ALIGN_FLOOR(start, align); 300 if ((end > addr) && (end < addr + len)) 301 end = RTE_ALIGN_CEIL(end, align); 302 } 303 DRV_LOG(DEBUG, 304 "port %u mempool %p using start=%p end=%p size=%zu for memory" 305 " region", 306 dev->data->port_id, (void *)mp, (void *)start, (void *)end, 307 (size_t)(end - start)); 308 mr->mr = ibv_reg_mr(priv->pd, (void *)start, end - start, 309 IBV_ACCESS_LOCAL_WRITE); 310 if (!mr->mr) { 311 rte_errno = ENOMEM; 312 return NULL; 313 } 314 mr->mp = mp; 315 mr->lkey = rte_cpu_to_be_32(mr->mr->lkey); 316 rte_atomic32_inc(&mr->refcnt); 317 DRV_LOG(DEBUG, "port %u new memory Region %p refcnt: %d", 318 dev->data->port_id, (void *)mr, rte_atomic32_read(&mr->refcnt)); 319 LIST_INSERT_HEAD(&priv->mr, mr, next); 320 return mr; 321 } 322 323 /** 324 * Search the memory region object in the memory region list. 325 * 326 * @param dev 327 * Pointer to Ethernet device. 328 * @param mp 329 * Pointer to the memory pool to register. 330 * 331 * @return 332 * The memory region on success. 333 */ 334 struct mlx5_mr * 335 mlx5_mr_get(struct rte_eth_dev *dev, struct rte_mempool *mp) 336 { 337 struct priv *priv = dev->data->dev_private; 338 struct mlx5_mr *mr; 339 340 assert(mp); 341 if (LIST_EMPTY(&priv->mr)) 342 return NULL; 343 LIST_FOREACH(mr, &priv->mr, next) { 344 if (mr->mp == mp) { 345 rte_atomic32_inc(&mr->refcnt); 346 DRV_LOG(DEBUG, "port %u memory region %p refcnt: %d", 347 dev->data->port_id, (void *)mr, 348 rte_atomic32_read(&mr->refcnt)); 349 return mr; 350 } 351 } 352 return NULL; 353 } 354 355 /** 356 * Release the memory region object. 357 * 358 * @param mr 359 * Pointer to memory region to release. 360 * 361 * @return 362 * 1 while a reference on it exists, 0 when freed. 363 */ 364 int 365 mlx5_mr_release(struct mlx5_mr *mr) 366 { 367 assert(mr); 368 DRV_LOG(DEBUG, "memory region %p refcnt: %d", (void *)mr, 369 rte_atomic32_read(&mr->refcnt)); 370 if (rte_atomic32_dec_and_test(&mr->refcnt)) { 371 claim_zero(ibv_dereg_mr(mr->mr)); 372 LIST_REMOVE(mr, next); 373 rte_free(mr); 374 return 0; 375 } 376 return 1; 377 } 378 379 /** 380 * Verify the flow list is empty 381 * 382 * @param dev 383 * Pointer to Ethernet device. 384 * 385 * @return 386 * The number of object not released. 387 */ 388 int 389 mlx5_mr_verify(struct rte_eth_dev *dev) 390 { 391 struct priv *priv = dev->data->dev_private; 392 int ret = 0; 393 struct mlx5_mr *mr; 394 395 LIST_FOREACH(mr, &priv->mr, next) { 396 DRV_LOG(DEBUG, "port %u memory region %p still referenced", 397 dev->data->port_id, (void *)mr); 398 ++ret; 399 } 400 return ret; 401 } 402