1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2019 Intel Corporation. 3 */ 4 #include <stdint.h> 5 #include <stdio.h> 6 #include <string.h> 7 #include <errno.h> 8 9 #include <rte_common.h> 10 #include <rte_lcore.h> 11 #include <rte_cycles.h> 12 #include <rte_eal.h> 13 #include <rte_log.h> 14 #include <rte_pci.h> 15 #include <rte_mbuf.h> 16 #include <rte_bus_pci.h> 17 #include <rte_memzone.h> 18 #include <rte_memcpy.h> 19 #include <rte_rawdev.h> 20 #include <rte_rawdev_pmd.h> 21 22 #include "ntb_hw_intel.h" 23 #include "rte_pmd_ntb.h" 24 #include "ntb.h" 25 26 static const struct rte_pci_id pci_id_ntb_map[] = { 27 { RTE_PCI_DEVICE(NTB_INTEL_VENDOR_ID, NTB_INTEL_DEV_ID_B2B_SKX) }, 28 { .vendor_id = 0, /* sentinel */ }, 29 }; 30 31 /* Align with enum ntb_xstats_idx */ 32 static struct rte_rawdev_xstats_name ntb_xstats_names[] = { 33 {"Tx-packets"}, 34 {"Tx-bytes"}, 35 {"Tx-errors"}, 36 {"Rx-packets"}, 37 {"Rx-bytes"}, 38 {"Rx-missed"}, 39 }; 40 #define NTB_XSTATS_NUM RTE_DIM(ntb_xstats_names) 41 42 static inline void 43 ntb_link_cleanup(struct rte_rawdev *dev) 44 { 45 struct ntb_hw *hw = dev->dev_private; 46 int status, i; 47 48 if (hw->ntb_ops->spad_write == NULL || 49 hw->ntb_ops->mw_set_trans == NULL) { 50 NTB_LOG(ERR, "Not supported to clean up link."); 51 return; 52 } 53 54 /* Clean spad registers. */ 55 for (i = 0; i < hw->spad_cnt; i++) { 56 status = (*hw->ntb_ops->spad_write)(dev, i, 0, 0); 57 if (status) 58 NTB_LOG(ERR, "Failed to clean local spad."); 59 } 60 61 /* Clear mw so that peer cannot access local memory.*/ 62 for (i = 0; i < hw->used_mw_num; i++) { 63 status = (*hw->ntb_ops->mw_set_trans)(dev, i, 0, 0); 64 if (status) 65 NTB_LOG(ERR, "Failed to clean mw."); 66 } 67 } 68 69 static inline int 70 ntb_handshake_work(const struct rte_rawdev *dev) 71 { 72 struct ntb_hw *hw = dev->dev_private; 73 uint32_t val; 74 int ret, i; 75 76 if (hw->ntb_ops->spad_write == NULL || 77 hw->ntb_ops->mw_set_trans == NULL) { 78 NTB_LOG(ERR, "Scratchpad/MW setting is not supported."); 79 return -ENOTSUP; 80 } 81 82 /* Tell peer the mw info of local side. */ 83 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_NUM_MWS, 1, hw->mw_cnt); 84 if (ret < 0) 85 return ret; 86 for (i = 0; i < hw->mw_cnt; i++) { 87 NTB_LOG(INFO, "Local %u mw size: 0x%"PRIx64"", i, 88 hw->mw_size[i]); 89 val = hw->mw_size[i] >> 32; 90 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_MW0_SZ_H + 2 * i, 91 1, val); 92 if (ret < 0) 93 return ret; 94 val = hw->mw_size[i]; 95 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_MW0_SZ_L + 2 * i, 96 1, val); 97 if (ret < 0) 98 return ret; 99 } 100 101 /* Tell peer about the queue info and map memory to the peer. */ 102 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_Q_SZ, 1, hw->queue_size); 103 if (ret < 0) 104 return ret; 105 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_NUM_QPS, 1, 106 hw->queue_pairs); 107 if (ret < 0) 108 return ret; 109 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_USED_MWS, 1, 110 hw->used_mw_num); 111 if (ret < 0) 112 return ret; 113 for (i = 0; i < hw->used_mw_num; i++) { 114 val = (uint64_t)(size_t)(hw->mz[i]->addr) >> 32; 115 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_MW0_BA_H + 2 * i, 116 1, val); 117 if (ret < 0) 118 return ret; 119 val = (uint64_t)(size_t)(hw->mz[i]->addr); 120 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_MW0_BA_L + 2 * i, 121 1, val); 122 if (ret < 0) 123 return ret; 124 } 125 126 for (i = 0; i < hw->used_mw_num; i++) { 127 ret = (*hw->ntb_ops->mw_set_trans)(dev, i, hw->mz[i]->iova, 128 hw->mz[i]->len); 129 if (ret < 0) 130 return ret; 131 } 132 133 /* Ring doorbell 0 to tell peer the device is ready. */ 134 ret = (*hw->ntb_ops->peer_db_set)(dev, 0); 135 if (ret < 0) 136 return ret; 137 138 return 0; 139 } 140 141 static void 142 ntb_dev_intr_handler(void *param) 143 { 144 struct rte_rawdev *dev = (struct rte_rawdev *)param; 145 struct ntb_hw *hw = dev->dev_private; 146 uint32_t val_h, val_l; 147 uint64_t peer_mw_size; 148 uint64_t db_bits = 0; 149 uint8_t peer_mw_cnt; 150 int i = 0; 151 152 if (hw->ntb_ops->db_read == NULL || 153 hw->ntb_ops->db_clear == NULL || 154 hw->ntb_ops->peer_db_set == NULL) { 155 NTB_LOG(ERR, "Doorbell is not supported."); 156 return; 157 } 158 159 db_bits = (*hw->ntb_ops->db_read)(dev); 160 if (!db_bits) 161 NTB_LOG(ERR, "No doorbells"); 162 163 /* Doorbell 0 is for peer device ready. */ 164 if (db_bits & 1) { 165 NTB_LOG(INFO, "DB0: Peer device is up."); 166 /* Clear received doorbell. */ 167 (*hw->ntb_ops->db_clear)(dev, 1); 168 169 /** 170 * Peer dev is already up. All mw settings are already done. 171 * Skip them. 172 */ 173 if (hw->peer_dev_up) 174 return; 175 176 if (hw->ntb_ops->spad_read == NULL) { 177 NTB_LOG(ERR, "Scratchpad read is not supported."); 178 return; 179 } 180 181 /* Check if mw setting on the peer is the same as local. */ 182 peer_mw_cnt = (*hw->ntb_ops->spad_read)(dev, SPAD_NUM_MWS, 0); 183 if (peer_mw_cnt != hw->mw_cnt) { 184 NTB_LOG(ERR, "Both mw cnt must be the same."); 185 return; 186 } 187 188 for (i = 0; i < hw->mw_cnt; i++) { 189 val_h = (*hw->ntb_ops->spad_read) 190 (dev, SPAD_MW0_SZ_H + 2 * i, 0); 191 val_l = (*hw->ntb_ops->spad_read) 192 (dev, SPAD_MW0_SZ_L + 2 * i, 0); 193 peer_mw_size = ((uint64_t)val_h << 32) | val_l; 194 NTB_LOG(DEBUG, "Peer %u mw size: 0x%"PRIx64"", i, 195 peer_mw_size); 196 if (peer_mw_size != hw->mw_size[i]) { 197 NTB_LOG(ERR, "Mw config must be the same."); 198 return; 199 } 200 } 201 202 hw->peer_dev_up = 1; 203 204 /** 205 * Handshake with peer. Spad_write & mw_set_trans only works 206 * when both devices are up. So write spad again when db is 207 * received. And set db again for the later device who may miss 208 * the 1st db. 209 */ 210 if (ntb_handshake_work(dev) < 0) { 211 NTB_LOG(ERR, "Handshake work failed."); 212 return; 213 } 214 215 /* To get the link info. */ 216 if (hw->ntb_ops->get_link_status == NULL) { 217 NTB_LOG(ERR, "Not supported to get link status."); 218 return; 219 } 220 (*hw->ntb_ops->get_link_status)(dev); 221 NTB_LOG(INFO, "Link is up. Link speed: %u. Link width: %u", 222 hw->link_speed, hw->link_width); 223 return; 224 } 225 226 if (db_bits & (1 << 1)) { 227 NTB_LOG(INFO, "DB1: Peer device is down."); 228 /* Clear received doorbell. */ 229 (*hw->ntb_ops->db_clear)(dev, 2); 230 231 /* Peer device will be down, So clean local side too. */ 232 ntb_link_cleanup(dev); 233 234 hw->peer_dev_up = 0; 235 /* Response peer's dev_stop request. */ 236 (*hw->ntb_ops->peer_db_set)(dev, 2); 237 return; 238 } 239 240 if (db_bits & (1 << 2)) { 241 NTB_LOG(INFO, "DB2: Peer device agrees dev to be down."); 242 /* Clear received doorbell. */ 243 (*hw->ntb_ops->db_clear)(dev, (1 << 2)); 244 hw->peer_dev_up = 0; 245 return; 246 } 247 } 248 249 static void 250 ntb_queue_conf_get(struct rte_rawdev *dev, 251 uint16_t queue_id, 252 rte_rawdev_obj_t queue_conf) 253 { 254 struct ntb_queue_conf *q_conf = queue_conf; 255 struct ntb_hw *hw = dev->dev_private; 256 257 q_conf->tx_free_thresh = hw->tx_queues[queue_id]->tx_free_thresh; 258 q_conf->nb_desc = hw->rx_queues[queue_id]->nb_rx_desc; 259 q_conf->rx_mp = hw->rx_queues[queue_id]->mpool; 260 } 261 262 static void 263 ntb_rxq_release_mbufs(struct ntb_rx_queue *q) 264 { 265 int i; 266 267 if (!q || !q->sw_ring) { 268 NTB_LOG(ERR, "Pointer to rxq or sw_ring is NULL"); 269 return; 270 } 271 272 for (i = 0; i < q->nb_rx_desc; i++) { 273 if (q->sw_ring[i].mbuf) { 274 rte_pktmbuf_free_seg(q->sw_ring[i].mbuf); 275 q->sw_ring[i].mbuf = NULL; 276 } 277 } 278 } 279 280 static void 281 ntb_rxq_release(struct ntb_rx_queue *rxq) 282 { 283 if (!rxq) { 284 NTB_LOG(ERR, "Pointer to rxq is NULL"); 285 return; 286 } 287 288 ntb_rxq_release_mbufs(rxq); 289 290 rte_free(rxq->sw_ring); 291 rte_free(rxq); 292 } 293 294 static int 295 ntb_rxq_setup(struct rte_rawdev *dev, 296 uint16_t qp_id, 297 rte_rawdev_obj_t queue_conf) 298 { 299 struct ntb_queue_conf *rxq_conf = queue_conf; 300 struct ntb_hw *hw = dev->dev_private; 301 struct ntb_rx_queue *rxq; 302 303 /* Allocate the rx queue data structure */ 304 rxq = rte_zmalloc_socket("ntb rx queue", 305 sizeof(struct ntb_rx_queue), 306 RTE_CACHE_LINE_SIZE, 307 dev->socket_id); 308 if (!rxq) { 309 NTB_LOG(ERR, "Failed to allocate memory for " 310 "rx queue data structure."); 311 return -ENOMEM; 312 } 313 314 if (rxq_conf->rx_mp == NULL) { 315 NTB_LOG(ERR, "Invalid null mempool pointer."); 316 return -EINVAL; 317 } 318 rxq->nb_rx_desc = rxq_conf->nb_desc; 319 rxq->mpool = rxq_conf->rx_mp; 320 rxq->port_id = dev->dev_id; 321 rxq->queue_id = qp_id; 322 rxq->hw = hw; 323 324 /* Allocate the software ring. */ 325 rxq->sw_ring = 326 rte_zmalloc_socket("ntb rx sw ring", 327 sizeof(struct ntb_rx_entry) * 328 rxq->nb_rx_desc, 329 RTE_CACHE_LINE_SIZE, 330 dev->socket_id); 331 if (!rxq->sw_ring) { 332 ntb_rxq_release(rxq); 333 rxq = NULL; 334 NTB_LOG(ERR, "Failed to allocate memory for SW ring"); 335 return -ENOMEM; 336 } 337 338 hw->rx_queues[qp_id] = rxq; 339 340 return 0; 341 } 342 343 static void 344 ntb_txq_release_mbufs(struct ntb_tx_queue *q) 345 { 346 int i; 347 348 if (!q || !q->sw_ring) { 349 NTB_LOG(ERR, "Pointer to txq or sw_ring is NULL"); 350 return; 351 } 352 353 for (i = 0; i < q->nb_tx_desc; i++) { 354 if (q->sw_ring[i].mbuf) { 355 rte_pktmbuf_free_seg(q->sw_ring[i].mbuf); 356 q->sw_ring[i].mbuf = NULL; 357 } 358 } 359 } 360 361 static void 362 ntb_txq_release(struct ntb_tx_queue *txq) 363 { 364 if (!txq) { 365 NTB_LOG(ERR, "Pointer to txq is NULL"); 366 return; 367 } 368 369 ntb_txq_release_mbufs(txq); 370 371 rte_free(txq->sw_ring); 372 rte_free(txq); 373 } 374 375 static int 376 ntb_txq_setup(struct rte_rawdev *dev, 377 uint16_t qp_id, 378 rte_rawdev_obj_t queue_conf) 379 { 380 struct ntb_queue_conf *txq_conf = queue_conf; 381 struct ntb_hw *hw = dev->dev_private; 382 struct ntb_tx_queue *txq; 383 uint16_t i, prev; 384 385 /* Allocate the TX queue data structure. */ 386 txq = rte_zmalloc_socket("ntb tx queue", 387 sizeof(struct ntb_tx_queue), 388 RTE_CACHE_LINE_SIZE, 389 dev->socket_id); 390 if (!txq) { 391 NTB_LOG(ERR, "Failed to allocate memory for " 392 "tx queue structure"); 393 return -ENOMEM; 394 } 395 396 txq->nb_tx_desc = txq_conf->nb_desc; 397 txq->port_id = dev->dev_id; 398 txq->queue_id = qp_id; 399 txq->hw = hw; 400 401 /* Allocate software ring */ 402 txq->sw_ring = 403 rte_zmalloc_socket("ntb tx sw ring", 404 sizeof(struct ntb_tx_entry) * 405 txq->nb_tx_desc, 406 RTE_CACHE_LINE_SIZE, 407 dev->socket_id); 408 if (!txq->sw_ring) { 409 ntb_txq_release(txq); 410 txq = NULL; 411 NTB_LOG(ERR, "Failed to allocate memory for SW TX ring"); 412 return -ENOMEM; 413 } 414 415 prev = txq->nb_tx_desc - 1; 416 for (i = 0; i < txq->nb_tx_desc; i++) { 417 txq->sw_ring[i].mbuf = NULL; 418 txq->sw_ring[i].last_id = i; 419 txq->sw_ring[prev].next_id = i; 420 prev = i; 421 } 422 423 txq->tx_free_thresh = txq_conf->tx_free_thresh ? 424 txq_conf->tx_free_thresh : 425 NTB_DFLT_TX_FREE_THRESH; 426 if (txq->tx_free_thresh >= txq->nb_tx_desc - 3) { 427 NTB_LOG(ERR, "tx_free_thresh must be less than nb_desc - 3. " 428 "(tx_free_thresh=%u qp_id=%u)", txq->tx_free_thresh, 429 qp_id); 430 return -EINVAL; 431 } 432 433 hw->tx_queues[qp_id] = txq; 434 435 return 0; 436 } 437 438 439 static int 440 ntb_queue_setup(struct rte_rawdev *dev, 441 uint16_t queue_id, 442 rte_rawdev_obj_t queue_conf) 443 { 444 struct ntb_hw *hw = dev->dev_private; 445 int ret; 446 447 if (queue_id >= hw->queue_pairs) 448 return -EINVAL; 449 450 ret = ntb_txq_setup(dev, queue_id, queue_conf); 451 if (ret < 0) 452 return ret; 453 454 ret = ntb_rxq_setup(dev, queue_id, queue_conf); 455 456 return ret; 457 } 458 459 static int 460 ntb_queue_release(struct rte_rawdev *dev, uint16_t queue_id) 461 { 462 struct ntb_hw *hw = dev->dev_private; 463 464 if (queue_id >= hw->queue_pairs) 465 return -EINVAL; 466 467 ntb_txq_release(hw->tx_queues[queue_id]); 468 hw->tx_queues[queue_id] = NULL; 469 ntb_rxq_release(hw->rx_queues[queue_id]); 470 hw->rx_queues[queue_id] = NULL; 471 472 return 0; 473 } 474 475 static uint16_t 476 ntb_queue_count(struct rte_rawdev *dev) 477 { 478 struct ntb_hw *hw = dev->dev_private; 479 return hw->queue_pairs; 480 } 481 482 static int 483 ntb_queue_init(struct rte_rawdev *dev, uint16_t qp_id) 484 { 485 struct ntb_hw *hw = dev->dev_private; 486 struct ntb_rx_queue *rxq = hw->rx_queues[qp_id]; 487 struct ntb_tx_queue *txq = hw->tx_queues[qp_id]; 488 volatile struct ntb_header *local_hdr; 489 struct ntb_header *remote_hdr; 490 uint16_t q_size = hw->queue_size; 491 uint32_t hdr_offset; 492 void *bar_addr; 493 uint16_t i; 494 495 if (hw->ntb_ops->get_peer_mw_addr == NULL) { 496 NTB_LOG(ERR, "Getting peer mw addr is not supported."); 497 return -EINVAL; 498 } 499 500 /* Put queue info into the start of shared memory. */ 501 hdr_offset = hw->hdr_size_per_queue * qp_id; 502 local_hdr = (volatile struct ntb_header *) 503 ((size_t)hw->mz[0]->addr + hdr_offset); 504 bar_addr = (*hw->ntb_ops->get_peer_mw_addr)(dev, 0); 505 if (bar_addr == NULL) 506 return -EINVAL; 507 remote_hdr = (struct ntb_header *) 508 ((size_t)bar_addr + hdr_offset); 509 510 /* rxq init. */ 511 rxq->rx_desc_ring = (struct ntb_desc *) 512 (&remote_hdr->desc_ring); 513 rxq->rx_used_ring = (volatile struct ntb_used *) 514 (&local_hdr->desc_ring[q_size]); 515 rxq->avail_cnt = &remote_hdr->avail_cnt; 516 rxq->used_cnt = &local_hdr->used_cnt; 517 518 for (i = 0; i < rxq->nb_rx_desc - 1; i++) { 519 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mpool); 520 if (unlikely(!mbuf)) { 521 NTB_LOG(ERR, "Failed to allocate mbuf for RX"); 522 return -ENOMEM; 523 } 524 mbuf->port = dev->dev_id; 525 526 rxq->sw_ring[i].mbuf = mbuf; 527 528 rxq->rx_desc_ring[i].addr = rte_pktmbuf_mtod(mbuf, size_t); 529 rxq->rx_desc_ring[i].len = mbuf->buf_len - RTE_PKTMBUF_HEADROOM; 530 } 531 rte_wmb(); 532 *rxq->avail_cnt = rxq->nb_rx_desc - 1; 533 rxq->last_avail = rxq->nb_rx_desc - 1; 534 rxq->last_used = 0; 535 536 /* txq init */ 537 txq->tx_desc_ring = (volatile struct ntb_desc *) 538 (&local_hdr->desc_ring); 539 txq->tx_used_ring = (struct ntb_used *) 540 (&remote_hdr->desc_ring[q_size]); 541 txq->avail_cnt = &local_hdr->avail_cnt; 542 txq->used_cnt = &remote_hdr->used_cnt; 543 544 rte_wmb(); 545 *txq->used_cnt = 0; 546 txq->last_used = 0; 547 txq->last_avail = 0; 548 txq->nb_tx_free = txq->nb_tx_desc - 1; 549 550 /* Set per queue stats. */ 551 for (i = 0; i < NTB_XSTATS_NUM; i++) { 552 hw->ntb_xstats[i + NTB_XSTATS_NUM * (qp_id + 1)] = 0; 553 hw->ntb_xstats_off[i + NTB_XSTATS_NUM * (qp_id + 1)] = 0; 554 } 555 556 return 0; 557 } 558 559 static inline void 560 ntb_enqueue_cleanup(struct ntb_tx_queue *txq) 561 { 562 struct ntb_tx_entry *sw_ring = txq->sw_ring; 563 uint16_t tx_free = txq->last_avail; 564 uint16_t nb_to_clean, i; 565 566 /* avail_cnt + 1 represents where to rx next in the peer. */ 567 nb_to_clean = (*txq->avail_cnt - txq->last_avail + 1 + 568 txq->nb_tx_desc) & (txq->nb_tx_desc - 1); 569 nb_to_clean = RTE_MIN(nb_to_clean, txq->tx_free_thresh); 570 for (i = 0; i < nb_to_clean; i++) { 571 if (sw_ring[tx_free].mbuf) 572 rte_pktmbuf_free_seg(sw_ring[tx_free].mbuf); 573 tx_free = (tx_free + 1) & (txq->nb_tx_desc - 1); 574 } 575 576 txq->nb_tx_free += nb_to_clean; 577 txq->last_avail = tx_free; 578 } 579 580 static int 581 ntb_enqueue_bufs(struct rte_rawdev *dev, 582 struct rte_rawdev_buf **buffers, 583 unsigned int count, 584 rte_rawdev_obj_t context) 585 { 586 struct ntb_hw *hw = dev->dev_private; 587 struct ntb_tx_queue *txq = hw->tx_queues[(size_t)context]; 588 struct ntb_tx_entry *sw_ring = txq->sw_ring; 589 struct rte_mbuf *txm; 590 struct ntb_used tx_used[NTB_MAX_DESC_SIZE]; 591 volatile struct ntb_desc *tx_item; 592 uint16_t tx_last, nb_segs, off, last_used, avail_cnt; 593 uint16_t nb_mbufs = 0; 594 uint16_t nb_tx = 0; 595 uint64_t bytes = 0; 596 void *buf_addr; 597 int i; 598 599 if (unlikely(hw->ntb_ops->ioremap == NULL)) { 600 NTB_LOG(ERR, "Ioremap not supported."); 601 return nb_tx; 602 } 603 604 if (unlikely(dev->started == 0 || hw->peer_dev_up == 0)) { 605 NTB_LOG(DEBUG, "Link is not up."); 606 return nb_tx; 607 } 608 609 if (txq->nb_tx_free < txq->tx_free_thresh) 610 ntb_enqueue_cleanup(txq); 611 612 off = NTB_XSTATS_NUM * ((size_t)context + 1); 613 last_used = txq->last_used; 614 avail_cnt = *txq->avail_cnt;/* Where to alloc next. */ 615 for (nb_tx = 0; nb_tx < count; nb_tx++) { 616 txm = (struct rte_mbuf *)(buffers[nb_tx]->buf_addr); 617 if (txm == NULL || txq->nb_tx_free < txm->nb_segs) 618 break; 619 620 tx_last = (txq->last_used + txm->nb_segs - 1) & 621 (txq->nb_tx_desc - 1); 622 nb_segs = txm->nb_segs; 623 for (i = 0; i < nb_segs; i++) { 624 /* Not enough ring space for tx. */ 625 if (txq->last_used == avail_cnt) 626 goto end_of_tx; 627 sw_ring[txq->last_used].mbuf = txm; 628 tx_item = txq->tx_desc_ring + txq->last_used; 629 630 if (!tx_item->len) { 631 (hw->ntb_xstats[NTB_TX_ERRS_ID + off])++; 632 goto end_of_tx; 633 } 634 if (txm->data_len > tx_item->len) { 635 NTB_LOG(ERR, "Data length exceeds buf length." 636 " Only %u data would be transmitted.", 637 tx_item->len); 638 txm->data_len = tx_item->len; 639 } 640 641 /* translate remote virtual addr to bar virtual addr */ 642 buf_addr = (*hw->ntb_ops->ioremap)(dev, tx_item->addr); 643 if (buf_addr == NULL) { 644 (hw->ntb_xstats[NTB_TX_ERRS_ID + off])++; 645 NTB_LOG(ERR, "Null remap addr."); 646 goto end_of_tx; 647 } 648 rte_memcpy(buf_addr, rte_pktmbuf_mtod(txm, void *), 649 txm->data_len); 650 651 tx_used[nb_mbufs].len = txm->data_len; 652 tx_used[nb_mbufs++].flags = (txq->last_used == 653 tx_last) ? 654 NTB_FLAG_EOP : 0; 655 656 /* update stats */ 657 bytes += txm->data_len; 658 659 txm = txm->next; 660 661 sw_ring[txq->last_used].next_id = (txq->last_used + 1) & 662 (txq->nb_tx_desc - 1); 663 sw_ring[txq->last_used].last_id = tx_last; 664 txq->last_used = (txq->last_used + 1) & 665 (txq->nb_tx_desc - 1); 666 } 667 txq->nb_tx_free -= nb_segs; 668 } 669 670 end_of_tx: 671 if (nb_tx) { 672 uint16_t nb1, nb2; 673 if (nb_mbufs > txq->nb_tx_desc - last_used) { 674 nb1 = txq->nb_tx_desc - last_used; 675 nb2 = nb_mbufs - txq->nb_tx_desc + last_used; 676 } else { 677 nb1 = nb_mbufs; 678 nb2 = 0; 679 } 680 rte_memcpy(txq->tx_used_ring + last_used, tx_used, 681 sizeof(struct ntb_used) * nb1); 682 rte_memcpy(txq->tx_used_ring, tx_used + nb1, 683 sizeof(struct ntb_used) * nb2); 684 rte_wmb(); 685 *txq->used_cnt = txq->last_used; 686 687 /* update queue stats */ 688 hw->ntb_xstats[NTB_TX_BYTES_ID + off] += bytes; 689 hw->ntb_xstats[NTB_TX_PKTS_ID + off] += nb_tx; 690 } 691 692 return nb_tx; 693 } 694 695 static int 696 ntb_dequeue_bufs(struct rte_rawdev *dev, 697 struct rte_rawdev_buf **buffers, 698 unsigned int count, 699 rte_rawdev_obj_t context) 700 { 701 struct ntb_hw *hw = dev->dev_private; 702 struct ntb_rx_queue *rxq = hw->rx_queues[(size_t)context]; 703 struct ntb_rx_entry *sw_ring = rxq->sw_ring; 704 struct ntb_desc rx_desc[NTB_MAX_DESC_SIZE]; 705 struct rte_mbuf *first, *rxm_t; 706 struct rte_mbuf *prev = NULL; 707 volatile struct ntb_used *rx_item; 708 uint16_t nb_mbufs = 0; 709 uint16_t nb_rx = 0; 710 uint64_t bytes = 0; 711 uint16_t off, last_avail, used_cnt, used_nb; 712 int i; 713 714 if (unlikely(dev->started == 0 || hw->peer_dev_up == 0)) { 715 NTB_LOG(DEBUG, "Link is not up"); 716 return nb_rx; 717 } 718 719 used_cnt = *rxq->used_cnt; 720 721 if (rxq->last_used == used_cnt) 722 return nb_rx; 723 724 last_avail = rxq->last_avail; 725 used_nb = (used_cnt - rxq->last_used) & (rxq->nb_rx_desc - 1); 726 count = RTE_MIN(count, used_nb); 727 for (nb_rx = 0; nb_rx < count; nb_rx++) { 728 i = 0; 729 while (true) { 730 rx_item = rxq->rx_used_ring + rxq->last_used; 731 rxm_t = sw_ring[rxq->last_used].mbuf; 732 rxm_t->data_len = rx_item->len; 733 rxm_t->data_off = RTE_PKTMBUF_HEADROOM; 734 rxm_t->port = rxq->port_id; 735 736 if (!i) { 737 rxm_t->nb_segs = 1; 738 first = rxm_t; 739 first->pkt_len = 0; 740 buffers[nb_rx]->buf_addr = rxm_t; 741 } else { 742 prev->next = rxm_t; 743 first->nb_segs++; 744 } 745 746 prev = rxm_t; 747 first->pkt_len += prev->data_len; 748 rxq->last_used = (rxq->last_used + 1) & 749 (rxq->nb_rx_desc - 1); 750 751 /* alloc new mbuf */ 752 rxm_t = rte_mbuf_raw_alloc(rxq->mpool); 753 if (unlikely(rxm_t == NULL)) { 754 NTB_LOG(ERR, "recv alloc mbuf failed."); 755 goto end_of_rx; 756 } 757 rxm_t->port = rxq->port_id; 758 sw_ring[rxq->last_avail].mbuf = rxm_t; 759 i++; 760 761 /* fill new desc */ 762 rx_desc[nb_mbufs].addr = 763 rte_pktmbuf_mtod(rxm_t, size_t); 764 rx_desc[nb_mbufs++].len = rxm_t->buf_len - 765 RTE_PKTMBUF_HEADROOM; 766 rxq->last_avail = (rxq->last_avail + 1) & 767 (rxq->nb_rx_desc - 1); 768 769 if (rx_item->flags & NTB_FLAG_EOP) 770 break; 771 } 772 /* update stats */ 773 bytes += first->pkt_len; 774 } 775 776 end_of_rx: 777 if (nb_rx) { 778 uint16_t nb1, nb2; 779 if (nb_mbufs > rxq->nb_rx_desc - last_avail) { 780 nb1 = rxq->nb_rx_desc - last_avail; 781 nb2 = nb_mbufs - rxq->nb_rx_desc + last_avail; 782 } else { 783 nb1 = nb_mbufs; 784 nb2 = 0; 785 } 786 rte_memcpy(rxq->rx_desc_ring + last_avail, rx_desc, 787 sizeof(struct ntb_desc) * nb1); 788 rte_memcpy(rxq->rx_desc_ring, rx_desc + nb1, 789 sizeof(struct ntb_desc) * nb2); 790 rte_wmb(); 791 *rxq->avail_cnt = rxq->last_avail; 792 793 /* update queue stats */ 794 off = NTB_XSTATS_NUM * ((size_t)context + 1); 795 hw->ntb_xstats[NTB_RX_BYTES_ID + off] += bytes; 796 hw->ntb_xstats[NTB_RX_PKTS_ID + off] += nb_rx; 797 hw->ntb_xstats[NTB_RX_MISS_ID + off] += (count - nb_rx); 798 } 799 800 return nb_rx; 801 } 802 803 static void 804 ntb_dev_info_get(struct rte_rawdev *dev, rte_rawdev_obj_t dev_info) 805 { 806 struct ntb_hw *hw = dev->dev_private; 807 struct ntb_dev_info *info = dev_info; 808 809 info->mw_cnt = hw->mw_cnt; 810 info->mw_size = hw->mw_size; 811 812 /** 813 * Intel hardware requires that mapped memory base address should be 814 * aligned with EMBARSZ and needs continuous memzone. 815 */ 816 info->mw_size_align = (uint8_t)(hw->pci_dev->id.vendor_id == 817 NTB_INTEL_VENDOR_ID); 818 819 if (!hw->queue_size || !hw->queue_pairs) { 820 NTB_LOG(ERR, "No queue size and queue num assigned."); 821 return; 822 } 823 824 hw->hdr_size_per_queue = RTE_ALIGN(sizeof(struct ntb_header) + 825 hw->queue_size * sizeof(struct ntb_desc) + 826 hw->queue_size * sizeof(struct ntb_used), 827 RTE_CACHE_LINE_SIZE); 828 info->ntb_hdr_size = hw->hdr_size_per_queue * hw->queue_pairs; 829 } 830 831 static int 832 ntb_dev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config) 833 { 834 struct ntb_dev_config *conf = config; 835 struct ntb_hw *hw = dev->dev_private; 836 uint32_t xstats_num; 837 int ret; 838 839 hw->queue_pairs = conf->num_queues; 840 hw->queue_size = conf->queue_size; 841 hw->used_mw_num = conf->mz_num; 842 hw->mz = conf->mz_list; 843 hw->rx_queues = rte_zmalloc("ntb_rx_queues", 844 sizeof(struct ntb_rx_queue *) * hw->queue_pairs, 0); 845 hw->tx_queues = rte_zmalloc("ntb_tx_queues", 846 sizeof(struct ntb_tx_queue *) * hw->queue_pairs, 0); 847 /* First total stats, then per queue stats. */ 848 xstats_num = (hw->queue_pairs + 1) * NTB_XSTATS_NUM; 849 hw->ntb_xstats = rte_zmalloc("ntb_xstats", xstats_num * 850 sizeof(uint64_t), 0); 851 hw->ntb_xstats_off = rte_zmalloc("ntb_xstats_off", xstats_num * 852 sizeof(uint64_t), 0); 853 854 /* Start handshake with the peer. */ 855 ret = ntb_handshake_work(dev); 856 if (ret < 0) { 857 rte_free(hw->rx_queues); 858 rte_free(hw->tx_queues); 859 hw->rx_queues = NULL; 860 hw->tx_queues = NULL; 861 return ret; 862 } 863 864 return 0; 865 } 866 867 static int 868 ntb_dev_start(struct rte_rawdev *dev) 869 { 870 struct ntb_hw *hw = dev->dev_private; 871 uint32_t peer_base_l, peer_val; 872 uint64_t peer_base_h; 873 uint32_t i; 874 int ret; 875 876 if (!hw->link_status || !hw->peer_dev_up) 877 return -EINVAL; 878 879 /* Set total stats. */ 880 for (i = 0; i < NTB_XSTATS_NUM; i++) { 881 hw->ntb_xstats[i] = 0; 882 hw->ntb_xstats_off[i] = 0; 883 } 884 885 for (i = 0; i < hw->queue_pairs; i++) { 886 ret = ntb_queue_init(dev, i); 887 if (ret) { 888 NTB_LOG(ERR, "Failed to init queue."); 889 goto err_q_init; 890 } 891 } 892 893 hw->peer_mw_base = rte_zmalloc("ntb_peer_mw_base", hw->mw_cnt * 894 sizeof(uint64_t), 0); 895 896 if (hw->ntb_ops->spad_read == NULL) { 897 ret = -ENOTSUP; 898 goto err_up; 899 } 900 901 peer_val = (*hw->ntb_ops->spad_read)(dev, SPAD_Q_SZ, 0); 902 if (peer_val != hw->queue_size) { 903 NTB_LOG(ERR, "Inconsistent queue size! (local: %u peer: %u)", 904 hw->queue_size, peer_val); 905 ret = -EINVAL; 906 goto err_up; 907 } 908 909 peer_val = (*hw->ntb_ops->spad_read)(dev, SPAD_NUM_QPS, 0); 910 if (peer_val != hw->queue_pairs) { 911 NTB_LOG(ERR, "Inconsistent number of queues! (local: %u peer:" 912 " %u)", hw->queue_pairs, peer_val); 913 ret = -EINVAL; 914 goto err_up; 915 } 916 917 hw->peer_used_mws = (*hw->ntb_ops->spad_read)(dev, SPAD_USED_MWS, 0); 918 919 for (i = 0; i < hw->peer_used_mws; i++) { 920 peer_base_h = (*hw->ntb_ops->spad_read)(dev, 921 SPAD_MW0_BA_H + 2 * i, 0); 922 peer_base_l = (*hw->ntb_ops->spad_read)(dev, 923 SPAD_MW0_BA_L + 2 * i, 0); 924 hw->peer_mw_base[i] = (peer_base_h << 32) + peer_base_l; 925 } 926 927 dev->started = 1; 928 929 return 0; 930 931 err_up: 932 rte_free(hw->peer_mw_base); 933 err_q_init: 934 for (i = 0; i < hw->queue_pairs; i++) { 935 ntb_rxq_release_mbufs(hw->rx_queues[i]); 936 ntb_txq_release_mbufs(hw->tx_queues[i]); 937 } 938 939 return ret; 940 } 941 942 static void 943 ntb_dev_stop(struct rte_rawdev *dev) 944 { 945 struct ntb_hw *hw = dev->dev_private; 946 uint32_t time_out; 947 int status, i; 948 949 if (!hw->peer_dev_up) 950 goto clean; 951 952 ntb_link_cleanup(dev); 953 954 /* Notify the peer that device will be down. */ 955 if (hw->ntb_ops->peer_db_set == NULL) { 956 NTB_LOG(ERR, "Peer doorbell setting is not supported."); 957 return; 958 } 959 status = (*hw->ntb_ops->peer_db_set)(dev, 1); 960 if (status) { 961 NTB_LOG(ERR, "Failed to tell peer device is down."); 962 return; 963 } 964 965 /* 966 * Set time out as 1s in case that the peer is stopped accidently 967 * without any notification. 968 */ 969 time_out = 1000000; 970 971 /* Wait for cleanup work down before db mask clear. */ 972 while (hw->peer_dev_up && time_out) { 973 time_out -= 10; 974 rte_delay_us(10); 975 } 976 977 clean: 978 /* Clear doorbells mask. */ 979 if (hw->ntb_ops->db_set_mask == NULL) { 980 NTB_LOG(ERR, "Doorbell mask setting is not supported."); 981 return; 982 } 983 status = (*hw->ntb_ops->db_set_mask)(dev, 984 (((uint64_t)1 << hw->db_cnt) - 1)); 985 if (status) 986 NTB_LOG(ERR, "Failed to clear doorbells."); 987 988 for (i = 0; i < hw->queue_pairs; i++) { 989 ntb_rxq_release_mbufs(hw->rx_queues[i]); 990 ntb_txq_release_mbufs(hw->tx_queues[i]); 991 } 992 993 dev->started = 0; 994 } 995 996 static int 997 ntb_dev_close(struct rte_rawdev *dev) 998 { 999 struct ntb_hw *hw = dev->dev_private; 1000 struct rte_intr_handle *intr_handle; 1001 int i; 1002 1003 if (dev->started) 1004 ntb_dev_stop(dev); 1005 1006 /* free queues */ 1007 for (i = 0; i < hw->queue_pairs; i++) 1008 ntb_queue_release(dev, i); 1009 hw->queue_pairs = 0; 1010 1011 intr_handle = &hw->pci_dev->intr_handle; 1012 /* Clean datapath event and vec mapping */ 1013 rte_intr_efd_disable(intr_handle); 1014 if (intr_handle->intr_vec) { 1015 rte_free(intr_handle->intr_vec); 1016 intr_handle->intr_vec = NULL; 1017 } 1018 /* Disable uio intr before callback unregister */ 1019 rte_intr_disable(intr_handle); 1020 1021 /* Unregister callback func to eal lib */ 1022 rte_intr_callback_unregister(intr_handle, 1023 ntb_dev_intr_handler, dev); 1024 1025 return 0; 1026 } 1027 1028 static int 1029 ntb_dev_reset(struct rte_rawdev *rawdev __rte_unused) 1030 { 1031 return 0; 1032 } 1033 1034 static int 1035 ntb_attr_set(struct rte_rawdev *dev, const char *attr_name, 1036 uint64_t attr_value) 1037 { 1038 struct ntb_hw *hw; 1039 int index; 1040 1041 if (dev == NULL || attr_name == NULL) { 1042 NTB_LOG(ERR, "Invalid arguments for setting attributes"); 1043 return -EINVAL; 1044 } 1045 1046 hw = dev->dev_private; 1047 1048 if (!strncmp(attr_name, NTB_SPAD_USER, NTB_SPAD_USER_LEN)) { 1049 if (hw->ntb_ops->spad_write == NULL) 1050 return -ENOTSUP; 1051 index = atoi(&attr_name[NTB_SPAD_USER_LEN]); 1052 (*hw->ntb_ops->spad_write)(dev, hw->spad_user_list[index], 1053 1, attr_value); 1054 NTB_LOG(DEBUG, "Set attribute (%s) Value (%" PRIu64 ")", 1055 attr_name, attr_value); 1056 return 0; 1057 } 1058 1059 if (!strncmp(attr_name, NTB_QUEUE_SZ_NAME, NTB_ATTR_NAME_LEN)) { 1060 hw->queue_size = attr_value; 1061 NTB_LOG(DEBUG, "Set attribute (%s) Value (%" PRIu64 ")", 1062 attr_name, attr_value); 1063 return 0; 1064 } 1065 1066 if (!strncmp(attr_name, NTB_QUEUE_NUM_NAME, NTB_ATTR_NAME_LEN)) { 1067 hw->queue_pairs = attr_value; 1068 NTB_LOG(DEBUG, "Set attribute (%s) Value (%" PRIu64 ")", 1069 attr_name, attr_value); 1070 return 0; 1071 } 1072 1073 /* Attribute not found. */ 1074 NTB_LOG(ERR, "Attribute not found."); 1075 return -EINVAL; 1076 } 1077 1078 static int 1079 ntb_attr_get(struct rte_rawdev *dev, const char *attr_name, 1080 uint64_t *attr_value) 1081 { 1082 struct ntb_hw *hw; 1083 int index; 1084 1085 if (dev == NULL || attr_name == NULL || attr_value == NULL) { 1086 NTB_LOG(ERR, "Invalid arguments for getting attributes"); 1087 return -EINVAL; 1088 } 1089 1090 hw = dev->dev_private; 1091 1092 if (!strncmp(attr_name, NTB_TOPO_NAME, NTB_ATTR_NAME_LEN)) { 1093 *attr_value = hw->topo; 1094 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")", 1095 attr_name, *attr_value); 1096 return 0; 1097 } 1098 1099 if (!strncmp(attr_name, NTB_LINK_STATUS_NAME, NTB_ATTR_NAME_LEN)) { 1100 /* hw->link_status only indicates hw link status. */ 1101 *attr_value = hw->link_status && hw->peer_dev_up; 1102 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")", 1103 attr_name, *attr_value); 1104 return 0; 1105 } 1106 1107 if (!strncmp(attr_name, NTB_SPEED_NAME, NTB_ATTR_NAME_LEN)) { 1108 *attr_value = hw->link_speed; 1109 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")", 1110 attr_name, *attr_value); 1111 return 0; 1112 } 1113 1114 if (!strncmp(attr_name, NTB_WIDTH_NAME, NTB_ATTR_NAME_LEN)) { 1115 *attr_value = hw->link_width; 1116 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")", 1117 attr_name, *attr_value); 1118 return 0; 1119 } 1120 1121 if (!strncmp(attr_name, NTB_MW_CNT_NAME, NTB_ATTR_NAME_LEN)) { 1122 *attr_value = hw->mw_cnt; 1123 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")", 1124 attr_name, *attr_value); 1125 return 0; 1126 } 1127 1128 if (!strncmp(attr_name, NTB_DB_CNT_NAME, NTB_ATTR_NAME_LEN)) { 1129 *attr_value = hw->db_cnt; 1130 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")", 1131 attr_name, *attr_value); 1132 return 0; 1133 } 1134 1135 if (!strncmp(attr_name, NTB_SPAD_CNT_NAME, NTB_ATTR_NAME_LEN)) { 1136 *attr_value = hw->spad_cnt; 1137 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")", 1138 attr_name, *attr_value); 1139 return 0; 1140 } 1141 1142 if (!strncmp(attr_name, NTB_SPAD_USER, NTB_SPAD_USER_LEN)) { 1143 if (hw->ntb_ops->spad_read == NULL) 1144 return -ENOTSUP; 1145 index = atoi(&attr_name[NTB_SPAD_USER_LEN]); 1146 *attr_value = (*hw->ntb_ops->spad_read)(dev, 1147 hw->spad_user_list[index], 0); 1148 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")", 1149 attr_name, *attr_value); 1150 return 0; 1151 } 1152 1153 /* Attribute not found. */ 1154 NTB_LOG(ERR, "Attribute not found."); 1155 return -EINVAL; 1156 } 1157 1158 static inline uint64_t 1159 ntb_stats_update(uint64_t offset, uint64_t stat) 1160 { 1161 if (stat >= offset) 1162 return (stat - offset); 1163 else 1164 return (uint64_t)(((uint64_t)-1) - offset + stat + 1); 1165 } 1166 1167 static int 1168 ntb_xstats_get(const struct rte_rawdev *dev, 1169 const unsigned int ids[], 1170 uint64_t values[], 1171 unsigned int n) 1172 { 1173 struct ntb_hw *hw = dev->dev_private; 1174 uint32_t i, j, off, xstats_num; 1175 1176 /* Calculate total stats of all queues. */ 1177 for (i = 0; i < NTB_XSTATS_NUM; i++) { 1178 hw->ntb_xstats[i] = 0; 1179 for (j = 0; j < hw->queue_pairs; j++) { 1180 off = NTB_XSTATS_NUM * (j + 1) + i; 1181 hw->ntb_xstats[i] += 1182 ntb_stats_update(hw->ntb_xstats_off[off], 1183 hw->ntb_xstats[off]); 1184 } 1185 } 1186 1187 xstats_num = NTB_XSTATS_NUM * (hw->queue_pairs + 1); 1188 for (i = 0; i < n && ids[i] < xstats_num; i++) { 1189 if (ids[i] < NTB_XSTATS_NUM) 1190 values[i] = hw->ntb_xstats[ids[i]]; 1191 else 1192 values[i] = 1193 ntb_stats_update(hw->ntb_xstats_off[ids[i]], 1194 hw->ntb_xstats[ids[i]]); 1195 } 1196 1197 return i; 1198 } 1199 1200 static int 1201 ntb_xstats_get_names(const struct rte_rawdev *dev, 1202 struct rte_rawdev_xstats_name *xstats_names, 1203 unsigned int size) 1204 { 1205 struct ntb_hw *hw = dev->dev_private; 1206 uint32_t xstats_num, i, j, off; 1207 1208 xstats_num = NTB_XSTATS_NUM * (hw->queue_pairs + 1); 1209 if (xstats_names == NULL || size < xstats_num) 1210 return xstats_num; 1211 1212 /* Total stats names */ 1213 memcpy(xstats_names, ntb_xstats_names, sizeof(ntb_xstats_names)); 1214 1215 /* Queue stats names */ 1216 for (i = 0; i < hw->queue_pairs; i++) { 1217 for (j = 0; j < NTB_XSTATS_NUM; j++) { 1218 off = j + (i + 1) * NTB_XSTATS_NUM; 1219 snprintf(xstats_names[off].name, 1220 sizeof(xstats_names[0].name), 1221 "%s_q%u", ntb_xstats_names[j].name, i); 1222 } 1223 } 1224 1225 return xstats_num; 1226 } 1227 1228 static uint64_t 1229 ntb_xstats_get_by_name(const struct rte_rawdev *dev, 1230 const char *name, unsigned int *id) 1231 { 1232 struct rte_rawdev_xstats_name *xstats_names; 1233 struct ntb_hw *hw = dev->dev_private; 1234 uint32_t xstats_num, i, j, off; 1235 1236 if (name == NULL) 1237 return -EINVAL; 1238 1239 xstats_num = NTB_XSTATS_NUM * (hw->queue_pairs + 1); 1240 xstats_names = rte_zmalloc("ntb_stats_name", 1241 sizeof(struct rte_rawdev_xstats_name) * 1242 xstats_num, 0); 1243 ntb_xstats_get_names(dev, xstats_names, xstats_num); 1244 1245 /* Calculate total stats of all queues. */ 1246 for (i = 0; i < NTB_XSTATS_NUM; i++) { 1247 for (j = 0; j < hw->queue_pairs; j++) { 1248 off = NTB_XSTATS_NUM * (j + 1) + i; 1249 hw->ntb_xstats[i] += 1250 ntb_stats_update(hw->ntb_xstats_off[off], 1251 hw->ntb_xstats[off]); 1252 } 1253 } 1254 1255 for (i = 0; i < xstats_num; i++) { 1256 if (!strncmp(name, xstats_names[i].name, 1257 RTE_RAW_DEV_XSTATS_NAME_SIZE)) { 1258 *id = i; 1259 rte_free(xstats_names); 1260 if (i < NTB_XSTATS_NUM) 1261 return hw->ntb_xstats[i]; 1262 else 1263 return ntb_stats_update(hw->ntb_xstats_off[i], 1264 hw->ntb_xstats[i]); 1265 } 1266 } 1267 1268 NTB_LOG(ERR, "Cannot find the xstats name."); 1269 1270 return -EINVAL; 1271 } 1272 1273 static int 1274 ntb_xstats_reset(struct rte_rawdev *dev, 1275 const uint32_t ids[], 1276 uint32_t nb_ids) 1277 { 1278 struct ntb_hw *hw = dev->dev_private; 1279 uint32_t i, j, off, xstats_num; 1280 1281 xstats_num = NTB_XSTATS_NUM * (hw->queue_pairs + 1); 1282 for (i = 0; i < nb_ids && ids[i] < xstats_num; i++) { 1283 if (ids[i] < NTB_XSTATS_NUM) { 1284 for (j = 0; j < hw->queue_pairs; j++) { 1285 off = NTB_XSTATS_NUM * (j + 1) + ids[i]; 1286 hw->ntb_xstats_off[off] = hw->ntb_xstats[off]; 1287 } 1288 } else { 1289 hw->ntb_xstats_off[ids[i]] = hw->ntb_xstats[ids[i]]; 1290 } 1291 } 1292 1293 return i; 1294 } 1295 1296 static const struct rte_rawdev_ops ntb_ops = { 1297 .dev_info_get = ntb_dev_info_get, 1298 .dev_configure = ntb_dev_configure, 1299 .dev_start = ntb_dev_start, 1300 .dev_stop = ntb_dev_stop, 1301 .dev_close = ntb_dev_close, 1302 .dev_reset = ntb_dev_reset, 1303 1304 .queue_def_conf = ntb_queue_conf_get, 1305 .queue_setup = ntb_queue_setup, 1306 .queue_release = ntb_queue_release, 1307 .queue_count = ntb_queue_count, 1308 1309 .enqueue_bufs = ntb_enqueue_bufs, 1310 .dequeue_bufs = ntb_dequeue_bufs, 1311 1312 .attr_get = ntb_attr_get, 1313 .attr_set = ntb_attr_set, 1314 1315 .xstats_get = ntb_xstats_get, 1316 .xstats_get_names = ntb_xstats_get_names, 1317 .xstats_get_by_name = ntb_xstats_get_by_name, 1318 .xstats_reset = ntb_xstats_reset, 1319 }; 1320 1321 static int 1322 ntb_init_hw(struct rte_rawdev *dev, struct rte_pci_device *pci_dev) 1323 { 1324 struct ntb_hw *hw = dev->dev_private; 1325 struct rte_intr_handle *intr_handle; 1326 int ret, i; 1327 1328 hw->pci_dev = pci_dev; 1329 hw->peer_dev_up = 0; 1330 hw->link_status = NTB_LINK_DOWN; 1331 hw->link_speed = NTB_SPEED_NONE; 1332 hw->link_width = NTB_WIDTH_NONE; 1333 1334 switch (pci_dev->id.device_id) { 1335 case NTB_INTEL_DEV_ID_B2B_SKX: 1336 hw->ntb_ops = &intel_ntb_ops; 1337 break; 1338 default: 1339 NTB_LOG(ERR, "Not supported device."); 1340 return -EINVAL; 1341 } 1342 1343 if (hw->ntb_ops->ntb_dev_init == NULL) 1344 return -ENOTSUP; 1345 ret = (*hw->ntb_ops->ntb_dev_init)(dev); 1346 if (ret) { 1347 NTB_LOG(ERR, "Unable to init ntb dev."); 1348 return ret; 1349 } 1350 1351 if (hw->ntb_ops->set_link == NULL) 1352 return -ENOTSUP; 1353 ret = (*hw->ntb_ops->set_link)(dev, 1); 1354 if (ret) 1355 return ret; 1356 1357 /* Init doorbell. */ 1358 hw->db_valid_mask = RTE_LEN2MASK(hw->db_cnt, uint64_t); 1359 1360 intr_handle = &pci_dev->intr_handle; 1361 /* Register callback func to eal lib */ 1362 rte_intr_callback_register(intr_handle, 1363 ntb_dev_intr_handler, dev); 1364 1365 ret = rte_intr_efd_enable(intr_handle, hw->db_cnt); 1366 if (ret) 1367 return ret; 1368 1369 /* To clarify, the interrupt for each doorbell is already mapped 1370 * by default for intel gen3. They are mapped to msix vec 1-32, 1371 * and hardware intr is mapped to 0. Map all to 0 for uio. 1372 */ 1373 if (!rte_intr_cap_multiple(intr_handle)) { 1374 for (i = 0; i < hw->db_cnt; i++) { 1375 if (hw->ntb_ops->vector_bind == NULL) 1376 return -ENOTSUP; 1377 ret = (*hw->ntb_ops->vector_bind)(dev, i, 0); 1378 if (ret) 1379 return ret; 1380 } 1381 } 1382 1383 if (hw->ntb_ops->db_set_mask == NULL || 1384 hw->ntb_ops->peer_db_set == NULL) { 1385 NTB_LOG(ERR, "Doorbell is not supported."); 1386 return -ENOTSUP; 1387 } 1388 hw->db_mask = 0; 1389 ret = (*hw->ntb_ops->db_set_mask)(dev, hw->db_mask); 1390 if (ret) { 1391 NTB_LOG(ERR, "Unable to enable intr for all dbs."); 1392 return ret; 1393 } 1394 1395 /* enable uio intr after callback register */ 1396 rte_intr_enable(intr_handle); 1397 1398 return ret; 1399 } 1400 1401 static int 1402 ntb_create(struct rte_pci_device *pci_dev, int socket_id) 1403 { 1404 char name[RTE_RAWDEV_NAME_MAX_LEN]; 1405 struct rte_rawdev *rawdev = NULL; 1406 int ret; 1407 1408 if (pci_dev == NULL) { 1409 NTB_LOG(ERR, "Invalid pci_dev."); 1410 return -EINVAL; 1411 } 1412 1413 memset(name, 0, sizeof(name)); 1414 snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "NTB:%x:%02x.%x", 1415 pci_dev->addr.bus, pci_dev->addr.devid, 1416 pci_dev->addr.function); 1417 1418 NTB_LOG(INFO, "Init %s on NUMA node %d", name, socket_id); 1419 1420 /* Allocate device structure. */ 1421 rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct ntb_hw), 1422 socket_id); 1423 if (rawdev == NULL) { 1424 NTB_LOG(ERR, "Unable to allocate rawdev."); 1425 return -EINVAL; 1426 } 1427 1428 rawdev->dev_ops = &ntb_ops; 1429 rawdev->device = &pci_dev->device; 1430 rawdev->driver_name = pci_dev->driver->driver.name; 1431 1432 ret = ntb_init_hw(rawdev, pci_dev); 1433 if (ret < 0) { 1434 NTB_LOG(ERR, "Unable to init ntb hw."); 1435 goto fail; 1436 } 1437 1438 return ret; 1439 1440 fail: 1441 if (rawdev != NULL) 1442 rte_rawdev_pmd_release(rawdev); 1443 1444 return ret; 1445 } 1446 1447 static int 1448 ntb_destroy(struct rte_pci_device *pci_dev) 1449 { 1450 char name[RTE_RAWDEV_NAME_MAX_LEN]; 1451 struct rte_rawdev *rawdev; 1452 int ret; 1453 1454 if (pci_dev == NULL) { 1455 NTB_LOG(ERR, "Invalid pci_dev."); 1456 ret = -EINVAL; 1457 return ret; 1458 } 1459 1460 memset(name, 0, sizeof(name)); 1461 snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "NTB:%x:%02x.%x", 1462 pci_dev->addr.bus, pci_dev->addr.devid, 1463 pci_dev->addr.function); 1464 1465 NTB_LOG(INFO, "Closing %s on NUMA node %d", name, rte_socket_id()); 1466 1467 rawdev = rte_rawdev_pmd_get_named_dev(name); 1468 if (rawdev == NULL) { 1469 NTB_LOG(ERR, "Invalid device name (%s)", name); 1470 ret = -EINVAL; 1471 return ret; 1472 } 1473 1474 ret = rte_rawdev_pmd_release(rawdev); 1475 if (ret) 1476 NTB_LOG(ERR, "Failed to destroy ntb rawdev."); 1477 1478 return ret; 1479 } 1480 1481 static int 1482 ntb_probe(struct rte_pci_driver *pci_drv __rte_unused, 1483 struct rte_pci_device *pci_dev) 1484 { 1485 return ntb_create(pci_dev, rte_socket_id()); 1486 } 1487 1488 static int 1489 ntb_remove(struct rte_pci_device *pci_dev) 1490 { 1491 return ntb_destroy(pci_dev); 1492 } 1493 1494 1495 static struct rte_pci_driver rte_ntb_pmd = { 1496 .id_table = pci_id_ntb_map, 1497 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_WC_ACTIVATE, 1498 .probe = ntb_probe, 1499 .remove = ntb_remove, 1500 }; 1501 1502 RTE_PMD_REGISTER_PCI(raw_ntb, rte_ntb_pmd); 1503 RTE_PMD_REGISTER_PCI_TABLE(raw_ntb, pci_id_ntb_map); 1504 RTE_PMD_REGISTER_KMOD_DEP(raw_ntb, "* igb_uio | uio_pci_generic | vfio-pci"); 1505 RTE_LOG_REGISTER(ntb_logtype, pmd.raw.ntb, INFO); 1506