1 /*- 2 * BSD LICENSE 3 * 4 * Copyright 2017 6WIND S.A. 5 * Copyright 2017 Mellanox 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of 6WIND S.A. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /** 35 * @file 36 * Memory management functions for mlx4 driver. 37 */ 38 39 #include <assert.h> 40 #include <errno.h> 41 #include <inttypes.h> 42 #include <stddef.h> 43 #include <stdint.h> 44 #include <string.h> 45 46 /* Verbs headers do not support -pedantic. */ 47 #ifdef PEDANTIC 48 #pragma GCC diagnostic ignored "-Wpedantic" 49 #endif 50 #include <infiniband/verbs.h> 51 #ifdef PEDANTIC 52 #pragma GCC diagnostic error "-Wpedantic" 53 #endif 54 55 #include <rte_branch_prediction.h> 56 #include <rte_common.h> 57 #include <rte_errno.h> 58 #include <rte_malloc.h> 59 #include <rte_memory.h> 60 #include <rte_mempool.h> 61 #include <rte_spinlock.h> 62 63 #include "mlx4_rxtx.h" 64 #include "mlx4_utils.h" 65 66 struct mlx4_check_mempool_data { 67 int ret; 68 char *start; 69 char *end; 70 }; 71 72 /** 73 * Called by mlx4_check_mempool() when iterating the memory chunks. 74 * 75 * @param[in] mp 76 * Pointer to memory pool (unused). 77 * @param[in, out] data 78 * Pointer to shared buffer with mlx4_check_mempool(). 79 * @param[in] memhdr 80 * Pointer to mempool chunk header. 81 * @param mem_idx 82 * Mempool element index (unused). 83 */ 84 static void 85 mlx4_check_mempool_cb(struct rte_mempool *mp, void *opaque, 86 struct rte_mempool_memhdr *memhdr, 87 unsigned int mem_idx) 88 { 89 struct mlx4_check_mempool_data *data = opaque; 90 91 (void)mp; 92 (void)mem_idx; 93 /* It already failed, skip the next chunks. */ 94 if (data->ret != 0) 95 return; 96 /* It is the first chunk. */ 97 if (data->start == NULL && data->end == NULL) { 98 data->start = memhdr->addr; 99 data->end = data->start + memhdr->len; 100 return; 101 } 102 if (data->end == memhdr->addr) { 103 data->end += memhdr->len; 104 return; 105 } 106 if (data->start == (char *)memhdr->addr + memhdr->len) { 107 data->start -= memhdr->len; 108 return; 109 } 110 /* Error, mempool is not virtually contiguous. */ 111 data->ret = -1; 112 } 113 114 /** 115 * Check if a mempool can be used: it must be virtually contiguous. 116 * 117 * @param[in] mp 118 * Pointer to memory pool. 119 * @param[out] start 120 * Pointer to the start address of the mempool virtual memory area. 121 * @param[out] end 122 * Pointer to the end address of the mempool virtual memory area. 123 * 124 * @return 125 * 0 on success (mempool is virtually contiguous), -1 on error. 126 */ 127 static int 128 mlx4_check_mempool(struct rte_mempool *mp, uintptr_t *start, uintptr_t *end) 129 { 130 struct mlx4_check_mempool_data data; 131 132 memset(&data, 0, sizeof(data)); 133 rte_mempool_mem_iter(mp, mlx4_check_mempool_cb, &data); 134 *start = (uintptr_t)data.start; 135 *end = (uintptr_t)data.end; 136 return data.ret; 137 } 138 139 /** 140 * Obtain a memory region from a memory pool. 141 * 142 * If a matching memory region already exists, it is returned with its 143 * reference count incremented, otherwise a new one is registered. 144 * 145 * @param priv 146 * Pointer to private structure. 147 * @param mp 148 * Pointer to memory pool. 149 * 150 * @return 151 * Memory region pointer, NULL in case of error and rte_errno is set. 152 */ 153 struct mlx4_mr * 154 mlx4_mr_get(struct priv *priv, struct rte_mempool *mp) 155 { 156 const struct rte_memseg *ms = rte_eal_get_physmem_layout(); 157 uintptr_t start; 158 uintptr_t end; 159 unsigned int i; 160 struct mlx4_mr *mr; 161 162 if (mlx4_check_mempool(mp, &start, &end) != 0) { 163 rte_errno = EINVAL; 164 ERROR("mempool %p: not virtually contiguous", 165 (void *)mp); 166 return NULL; 167 } 168 DEBUG("mempool %p area start=%p end=%p size=%zu", 169 (void *)mp, (void *)start, (void *)end, 170 (size_t)(end - start)); 171 /* Round start and end to page boundary if found in memory segments. */ 172 for (i = 0; (i < RTE_MAX_MEMSEG) && (ms[i].addr != NULL); ++i) { 173 uintptr_t addr = (uintptr_t)ms[i].addr; 174 size_t len = ms[i].len; 175 unsigned int align = ms[i].hugepage_sz; 176 177 if ((start > addr) && (start < addr + len)) 178 start = RTE_ALIGN_FLOOR(start, align); 179 if ((end > addr) && (end < addr + len)) 180 end = RTE_ALIGN_CEIL(end, align); 181 } 182 DEBUG("mempool %p using start=%p end=%p size=%zu for MR", 183 (void *)mp, (void *)start, (void *)end, 184 (size_t)(end - start)); 185 rte_spinlock_lock(&priv->mr_lock); 186 LIST_FOREACH(mr, &priv->mr, next) 187 if (mp == mr->mp && start >= mr->start && end <= mr->end) 188 break; 189 if (mr) { 190 ++mr->refcnt; 191 goto release; 192 } 193 mr = rte_malloc(__func__, sizeof(*mr), 0); 194 if (!mr) { 195 rte_errno = ENOMEM; 196 goto release; 197 } 198 *mr = (struct mlx4_mr){ 199 .start = start, 200 .end = end, 201 .refcnt = 1, 202 .priv = priv, 203 .mr = ibv_reg_mr(priv->pd, (void *)start, end - start, 204 IBV_ACCESS_LOCAL_WRITE), 205 .mp = mp, 206 }; 207 if (mr->mr) { 208 mr->lkey = mr->mr->lkey; 209 LIST_INSERT_HEAD(&priv->mr, mr, next); 210 } else { 211 rte_free(mr); 212 mr = NULL; 213 rte_errno = errno ? errno : EINVAL; 214 } 215 release: 216 rte_spinlock_unlock(&priv->mr_lock); 217 return mr; 218 } 219 220 /** 221 * Release a memory region. 222 * 223 * This function decrements its reference count and destroys it after 224 * reaching 0. 225 * 226 * Note to avoid race conditions given this function may be used from the 227 * data plane, it's extremely important that each user holds its own 228 * reference. 229 * 230 * @param mr 231 * Memory region to release. 232 */ 233 void 234 mlx4_mr_put(struct mlx4_mr *mr) 235 { 236 struct priv *priv = mr->priv; 237 238 rte_spinlock_lock(&priv->mr_lock); 239 assert(mr->refcnt); 240 if (--mr->refcnt) 241 goto release; 242 LIST_REMOVE(mr, next); 243 claim_zero(ibv_dereg_mr(mr->mr)); 244 rte_free(mr); 245 release: 246 rte_spinlock_unlock(&priv->mr_lock); 247 } 248 249 /** 250 * Add memory region (MR) <-> memory pool (MP) association to txq->mp2mr[]. 251 * If mp2mr[] is full, remove an entry first. 252 * 253 * @param txq 254 * Pointer to Tx queue structure. 255 * @param[in] mp 256 * Memory pool for which a memory region lkey must be added. 257 * @param[in] i 258 * Index in memory pool (MP) where to add memory region (MR). 259 * 260 * @return 261 * Added mr->lkey on success, (uint32_t)-1 on failure. 262 */ 263 uint32_t 264 mlx4_txq_add_mr(struct txq *txq, struct rte_mempool *mp, uint32_t i) 265 { 266 struct mlx4_mr *mr; 267 268 /* Add a new entry, register MR first. */ 269 DEBUG("%p: discovered new memory pool \"%s\" (%p)", 270 (void *)txq, mp->name, (void *)mp); 271 mr = mlx4_mr_get(txq->priv, mp); 272 if (unlikely(mr == NULL)) { 273 DEBUG("%p: unable to configure MR, mlx4_mr_get() failed", 274 (void *)txq); 275 return (uint32_t)-1; 276 } 277 if (unlikely(i == RTE_DIM(txq->mp2mr))) { 278 /* Table is full, remove oldest entry. */ 279 DEBUG("%p: MR <-> MP table full, dropping oldest entry.", 280 (void *)txq); 281 --i; 282 mlx4_mr_put(txq->mp2mr[0].mr); 283 memmove(&txq->mp2mr[0], &txq->mp2mr[1], 284 (sizeof(txq->mp2mr) - sizeof(txq->mp2mr[0]))); 285 } 286 /* Store the new entry. */ 287 txq->mp2mr[i].mp = mp; 288 txq->mp2mr[i].mr = mr; 289 txq->mp2mr[i].lkey = mr->lkey; 290 DEBUG("%p: new MR lkey for MP \"%s\" (%p): 0x%08" PRIu32, 291 (void *)txq, mp->name, (void *)mp, txq->mp2mr[i].lkey); 292 return txq->mp2mr[i].lkey; 293 } 294