1 /*- 2 * BSD LICENSE 3 * 4 * Copyright 2017 6WIND S.A. 5 * Copyright 2017 Mellanox 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of 6WIND S.A. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /** 35 * @file 36 * Interrupts handling for mlx4 driver. 37 */ 38 39 #include <assert.h> 40 #include <errno.h> 41 #include <stdint.h> 42 #include <stdlib.h> 43 44 /* Verbs headers do not support -pedantic. */ 45 #ifdef PEDANTIC 46 #pragma GCC diagnostic ignored "-Wpedantic" 47 #endif 48 #include <infiniband/verbs.h> 49 #ifdef PEDANTIC 50 #pragma GCC diagnostic error "-Wpedantic" 51 #endif 52 53 #include <rte_alarm.h> 54 #include <rte_errno.h> 55 #include <rte_ethdev.h> 56 #include <rte_io.h> 57 #include <rte_interrupts.h> 58 59 #include "mlx4.h" 60 #include "mlx4_rxtx.h" 61 #include "mlx4_utils.h" 62 63 static int mlx4_link_status_check(struct priv *priv); 64 65 /** 66 * Clean up Rx interrupts handler. 67 * 68 * @param priv 69 * Pointer to private structure. 70 */ 71 static void 72 mlx4_rx_intr_vec_disable(struct priv *priv) 73 { 74 struct rte_intr_handle *intr_handle = &priv->intr_handle; 75 76 rte_intr_free_epoll_fd(intr_handle); 77 free(intr_handle->intr_vec); 78 intr_handle->nb_efd = 0; 79 intr_handle->intr_vec = NULL; 80 } 81 82 /** 83 * Allocate queue vector and fill epoll fd list for Rx interrupts. 84 * 85 * @param priv 86 * Pointer to private structure. 87 * 88 * @return 89 * 0 on success, negative errno value otherwise and rte_errno is set. 90 */ 91 static int 92 mlx4_rx_intr_vec_enable(struct priv *priv) 93 { 94 unsigned int i; 95 unsigned int rxqs_n = priv->dev->data->nb_rx_queues; 96 unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID); 97 unsigned int count = 0; 98 struct rte_intr_handle *intr_handle = &priv->intr_handle; 99 100 mlx4_rx_intr_vec_disable(priv); 101 intr_handle->intr_vec = malloc(n * sizeof(intr_handle->intr_vec[0])); 102 if (intr_handle->intr_vec == NULL) { 103 rte_errno = ENOMEM; 104 ERROR("failed to allocate memory for interrupt vector," 105 " Rx interrupts will not be supported"); 106 return -rte_errno; 107 } 108 for (i = 0; i != n; ++i) { 109 struct rxq *rxq = priv->dev->data->rx_queues[i]; 110 111 /* Skip queues that cannot request interrupts. */ 112 if (!rxq || !rxq->channel) { 113 /* Use invalid intr_vec[] index to disable entry. */ 114 intr_handle->intr_vec[i] = 115 RTE_INTR_VEC_RXTX_OFFSET + 116 RTE_MAX_RXTX_INTR_VEC_ID; 117 continue; 118 } 119 if (count >= RTE_MAX_RXTX_INTR_VEC_ID) { 120 rte_errno = E2BIG; 121 ERROR("too many Rx queues for interrupt vector size" 122 " (%d), Rx interrupts cannot be enabled", 123 RTE_MAX_RXTX_INTR_VEC_ID); 124 mlx4_rx_intr_vec_disable(priv); 125 return -rte_errno; 126 } 127 intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count; 128 intr_handle->efds[count] = rxq->channel->fd; 129 count++; 130 } 131 if (!count) 132 mlx4_rx_intr_vec_disable(priv); 133 else 134 intr_handle->nb_efd = count; 135 return 0; 136 } 137 138 /** 139 * Process scheduled link status check. 140 * 141 * If LSC interrupts are requested, process related callback. 142 * 143 * @param priv 144 * Pointer to private structure. 145 */ 146 static void 147 mlx4_link_status_alarm(struct priv *priv) 148 { 149 const struct rte_intr_conf *const intr_conf = 150 &priv->dev->data->dev_conf.intr_conf; 151 152 assert(priv->intr_alarm == 1); 153 priv->intr_alarm = 0; 154 if (intr_conf->lsc && !mlx4_link_status_check(priv)) 155 _rte_eth_dev_callback_process(priv->dev, 156 RTE_ETH_EVENT_INTR_LSC, 157 NULL, NULL); 158 } 159 160 /** 161 * Check link status. 162 * 163 * In case of inconsistency, another check is scheduled. 164 * 165 * @param priv 166 * Pointer to private structure. 167 * 168 * @return 169 * 0 on success (link status is consistent), negative errno value 170 * otherwise and rte_errno is set. 171 */ 172 static int 173 mlx4_link_status_check(struct priv *priv) 174 { 175 struct rte_eth_link *link = &priv->dev->data->dev_link; 176 int ret = mlx4_link_update(priv->dev, 0); 177 178 if (ret) 179 return ret; 180 if ((!link->link_speed && link->link_status) || 181 (link->link_speed && !link->link_status)) { 182 if (!priv->intr_alarm) { 183 /* Inconsistent status, check again later. */ 184 ret = rte_eal_alarm_set(MLX4_INTR_ALARM_TIMEOUT, 185 (void (*)(void *)) 186 mlx4_link_status_alarm, 187 priv); 188 if (ret) 189 return ret; 190 priv->intr_alarm = 1; 191 } 192 rte_errno = EINPROGRESS; 193 return -rte_errno; 194 } 195 return 0; 196 } 197 198 /** 199 * Handle interrupts from the NIC. 200 * 201 * @param priv 202 * Pointer to private structure. 203 */ 204 static void 205 mlx4_interrupt_handler(struct priv *priv) 206 { 207 enum { LSC, RMV, }; 208 static const enum rte_eth_event_type type[] = { 209 [LSC] = RTE_ETH_EVENT_INTR_LSC, 210 [RMV] = RTE_ETH_EVENT_INTR_RMV, 211 }; 212 uint32_t caught[RTE_DIM(type)] = { 0 }; 213 struct ibv_async_event event; 214 const struct rte_intr_conf *const intr_conf = 215 &priv->dev->data->dev_conf.intr_conf; 216 unsigned int i; 217 218 /* Read all message and acknowledge them. */ 219 while (!ibv_get_async_event(priv->ctx, &event)) { 220 switch (event.event_type) { 221 case IBV_EVENT_PORT_ACTIVE: 222 case IBV_EVENT_PORT_ERR: 223 if (intr_conf->lsc && !mlx4_link_status_check(priv)) 224 ++caught[LSC]; 225 break; 226 case IBV_EVENT_DEVICE_FATAL: 227 if (intr_conf->rmv) 228 ++caught[RMV]; 229 break; 230 default: 231 DEBUG("event type %d on physical port %d not handled", 232 event.event_type, event.element.port_num); 233 } 234 ibv_ack_async_event(&event); 235 } 236 for (i = 0; i != RTE_DIM(caught); ++i) 237 if (caught[i]) 238 _rte_eth_dev_callback_process(priv->dev, type[i], 239 NULL, NULL); 240 } 241 242 /** 243 * MLX4 CQ notification . 244 * 245 * @param rxq 246 * Pointer to receive queue structure. 247 * @param solicited 248 * Is request solicited or not. 249 */ 250 static void 251 mlx4_arm_cq(struct rxq *rxq, int solicited) 252 { 253 struct mlx4_cq *cq = &rxq->mcq; 254 uint64_t doorbell; 255 uint32_t sn = cq->arm_sn & MLX4_CQ_DB_GEQ_N_MASK; 256 uint32_t ci = cq->cons_index & MLX4_CQ_DB_CI_MASK; 257 uint32_t cmd = solicited ? MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT; 258 259 *cq->arm_db = rte_cpu_to_be_32(sn << 28 | cmd | ci); 260 /* 261 * Make sure that the doorbell record in host memory is 262 * written before ringing the doorbell via PCI MMIO. 263 */ 264 rte_wmb(); 265 doorbell = sn << 28 | cmd | cq->cqn; 266 doorbell <<= 32; 267 doorbell |= ci; 268 rte_write64(rte_cpu_to_be_64(doorbell), cq->cq_db_reg); 269 } 270 271 /** 272 * Uninstall interrupt handler. 273 * 274 * @param priv 275 * Pointer to private structure. 276 * 277 * @return 278 * 0 on success, negative errno value otherwise and rte_errno is set. 279 */ 280 int 281 mlx4_intr_uninstall(struct priv *priv) 282 { 283 int err = rte_errno; /* Make sure rte_errno remains unchanged. */ 284 285 if (priv->intr_handle.fd != -1) { 286 rte_intr_callback_unregister(&priv->intr_handle, 287 (void (*)(void *)) 288 mlx4_interrupt_handler, 289 priv); 290 priv->intr_handle.fd = -1; 291 } 292 rte_eal_alarm_cancel((void (*)(void *))mlx4_link_status_alarm, priv); 293 priv->intr_alarm = 0; 294 mlx4_rxq_intr_disable(priv); 295 rte_errno = err; 296 return 0; 297 } 298 299 /** 300 * Install interrupt handler. 301 * 302 * @param priv 303 * Pointer to private structure. 304 * 305 * @return 306 * 0 on success, negative errno value otherwise and rte_errno is set. 307 */ 308 int 309 mlx4_intr_install(struct priv *priv) 310 { 311 const struct rte_intr_conf *const intr_conf = 312 &priv->dev->data->dev_conf.intr_conf; 313 int rc; 314 315 mlx4_intr_uninstall(priv); 316 if (intr_conf->lsc | intr_conf->rmv) { 317 priv->intr_handle.fd = priv->ctx->async_fd; 318 rc = rte_intr_callback_register(&priv->intr_handle, 319 (void (*)(void *)) 320 mlx4_interrupt_handler, 321 priv); 322 if (rc < 0) { 323 rte_errno = -rc; 324 goto error; 325 } 326 } 327 return 0; 328 error: 329 mlx4_intr_uninstall(priv); 330 return -rte_errno; 331 } 332 333 /** 334 * DPDK callback for Rx queue interrupt disable. 335 * 336 * @param dev 337 * Pointer to Ethernet device structure. 338 * @param idx 339 * Rx queue index. 340 * 341 * @return 342 * 0 on success, negative errno value otherwise and rte_errno is set. 343 */ 344 int 345 mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx) 346 { 347 struct rxq *rxq = dev->data->rx_queues[idx]; 348 struct ibv_cq *ev_cq; 349 void *ev_ctx; 350 int ret; 351 352 if (!rxq || !rxq->channel) { 353 ret = EINVAL; 354 } else { 355 ret = ibv_get_cq_event(rxq->cq->channel, &ev_cq, &ev_ctx); 356 if (ret || ev_cq != rxq->cq) 357 ret = EINVAL; 358 } 359 if (ret) { 360 rte_errno = ret; 361 WARN("unable to disable interrupt on rx queue %d", 362 idx); 363 } else { 364 rxq->mcq.arm_sn++; 365 ibv_ack_cq_events(rxq->cq, 1); 366 } 367 return -ret; 368 } 369 370 /** 371 * DPDK callback for Rx queue interrupt enable. 372 * 373 * @param dev 374 * Pointer to Ethernet device structure. 375 * @param idx 376 * Rx queue index. 377 * 378 * @return 379 * 0 on success, negative errno value otherwise and rte_errno is set. 380 */ 381 int 382 mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx) 383 { 384 struct rxq *rxq = dev->data->rx_queues[idx]; 385 int ret = 0; 386 387 if (!rxq || !rxq->channel) { 388 ret = EINVAL; 389 rte_errno = ret; 390 WARN("unable to arm interrupt on rx queue %d", idx); 391 } else { 392 mlx4_arm_cq(rxq, 0); 393 } 394 return -ret; 395 } 396 397 /** 398 * Enable datapath interrupts. 399 * 400 * @param priv 401 * Pointer to private structure. 402 * 403 * @return 404 * 0 on success, negative errno value otherwise and rte_errno is set. 405 */ 406 int 407 mlx4_rxq_intr_enable(struct priv *priv) 408 { 409 const struct rte_intr_conf *const intr_conf = 410 &priv->dev->data->dev_conf.intr_conf; 411 412 if (intr_conf->rxq && mlx4_rx_intr_vec_enable(priv) < 0) 413 goto error; 414 return 0; 415 error: 416 return -rte_errno; 417 } 418 419 /** 420 * Disable datapath interrupts, keeping other interrupts intact. 421 * 422 * @param priv 423 * Pointer to private structure. 424 */ 425 void 426 mlx4_rxq_intr_disable(struct priv *priv) 427 { 428 int err = rte_errno; /* Make sure rte_errno remains unchanged. */ 429 430 mlx4_rx_intr_vec_disable(priv); 431 rte_errno = err; 432 } 433