1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2019 Intel Corporation. 3 */ 4 #include <stdint.h> 5 #include <stdio.h> 6 #include <string.h> 7 #include <errno.h> 8 9 #include <rte_common.h> 10 #include <rte_lcore.h> 11 #include <rte_cycles.h> 12 #include <rte_eal.h> 13 #include <rte_log.h> 14 #include <rte_pci.h> 15 #include <rte_mbuf.h> 16 #include <rte_bus_pci.h> 17 #include <rte_memzone.h> 18 #include <rte_memcpy.h> 19 #include <rte_rawdev.h> 20 #include <rte_rawdev_pmd.h> 21 22 #include "ntb_hw_intel.h" 23 #include "rte_pmd_ntb.h" 24 #include "ntb.h" 25 26 int ntb_logtype; 27 28 static const struct rte_pci_id pci_id_ntb_map[] = { 29 { RTE_PCI_DEVICE(NTB_INTEL_VENDOR_ID, NTB_INTEL_DEV_ID_B2B_SKX) }, 30 { .vendor_id = 0, /* sentinel */ }, 31 }; 32 33 /* Align with enum ntb_xstats_idx */ 34 static struct rte_rawdev_xstats_name ntb_xstats_names[] = { 35 {"Tx-packets"}, 36 {"Tx-bytes"}, 37 {"Tx-errors"}, 38 {"Rx-packets"}, 39 {"Rx-bytes"}, 40 {"Rx-missed"}, 41 }; 42 #define NTB_XSTATS_NUM RTE_DIM(ntb_xstats_names) 43 44 static inline void 45 ntb_link_cleanup(struct rte_rawdev *dev) 46 { 47 struct ntb_hw *hw = dev->dev_private; 48 int status, i; 49 50 if (hw->ntb_ops->spad_write == NULL || 51 hw->ntb_ops->mw_set_trans == NULL) { 52 NTB_LOG(ERR, "Not supported to clean up link."); 53 return; 54 } 55 56 /* Clean spad registers. */ 57 for (i = 0; i < hw->spad_cnt; i++) { 58 status = (*hw->ntb_ops->spad_write)(dev, i, 0, 0); 59 if (status) 60 NTB_LOG(ERR, "Failed to clean local spad."); 61 } 62 63 /* Clear mw so that peer cannot access local memory.*/ 64 for (i = 0; i < hw->used_mw_num; i++) { 65 status = (*hw->ntb_ops->mw_set_trans)(dev, i, 0, 0); 66 if (status) 67 NTB_LOG(ERR, "Failed to clean mw."); 68 } 69 } 70 71 static inline int 72 ntb_handshake_work(const struct rte_rawdev *dev) 73 { 74 struct ntb_hw *hw = dev->dev_private; 75 uint32_t val; 76 int ret, i; 77 78 if (hw->ntb_ops->spad_write == NULL || 79 hw->ntb_ops->mw_set_trans == NULL) { 80 NTB_LOG(ERR, "Scratchpad/MW setting is not supported."); 81 return -ENOTSUP; 82 } 83 84 /* Tell peer the mw info of local side. */ 85 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_NUM_MWS, 1, hw->mw_cnt); 86 if (ret < 0) 87 return ret; 88 for (i = 0; i < hw->mw_cnt; i++) { 89 NTB_LOG(INFO, "Local %u mw size: 0x%"PRIx64"", i, 90 hw->mw_size[i]); 91 val = hw->mw_size[i] >> 32; 92 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_MW0_SZ_H + 2 * i, 93 1, val); 94 if (ret < 0) 95 return ret; 96 val = hw->mw_size[i]; 97 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_MW0_SZ_L + 2 * i, 98 1, val); 99 if (ret < 0) 100 return ret; 101 } 102 103 /* Tell peer about the queue info and map memory to the peer. */ 104 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_Q_SZ, 1, hw->queue_size); 105 if (ret < 0) 106 return ret; 107 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_NUM_QPS, 1, 108 hw->queue_pairs); 109 if (ret < 0) 110 return ret; 111 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_USED_MWS, 1, 112 hw->used_mw_num); 113 if (ret < 0) 114 return ret; 115 for (i = 0; i < hw->used_mw_num; i++) { 116 val = (uint64_t)(size_t)(hw->mz[i]->addr) >> 32; 117 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_MW0_BA_H + 2 * i, 118 1, val); 119 if (ret < 0) 120 return ret; 121 val = (uint64_t)(size_t)(hw->mz[i]->addr); 122 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_MW0_BA_L + 2 * i, 123 1, val); 124 if (ret < 0) 125 return ret; 126 } 127 128 for (i = 0; i < hw->used_mw_num; i++) { 129 ret = (*hw->ntb_ops->mw_set_trans)(dev, i, hw->mz[i]->iova, 130 hw->mz[i]->len); 131 if (ret < 0) 132 return ret; 133 } 134 135 /* Ring doorbell 0 to tell peer the device is ready. */ 136 ret = (*hw->ntb_ops->peer_db_set)(dev, 0); 137 if (ret < 0) 138 return ret; 139 140 return 0; 141 } 142 143 static void 144 ntb_dev_intr_handler(void *param) 145 { 146 struct rte_rawdev *dev = (struct rte_rawdev *)param; 147 struct ntb_hw *hw = dev->dev_private; 148 uint32_t val_h, val_l; 149 uint64_t peer_mw_size; 150 uint64_t db_bits = 0; 151 uint8_t peer_mw_cnt; 152 int i = 0; 153 154 if (hw->ntb_ops->db_read == NULL || 155 hw->ntb_ops->db_clear == NULL || 156 hw->ntb_ops->peer_db_set == NULL) { 157 NTB_LOG(ERR, "Doorbell is not supported."); 158 return; 159 } 160 161 db_bits = (*hw->ntb_ops->db_read)(dev); 162 if (!db_bits) 163 NTB_LOG(ERR, "No doorbells"); 164 165 /* Doorbell 0 is for peer device ready. */ 166 if (db_bits & 1) { 167 NTB_LOG(INFO, "DB0: Peer device is up."); 168 /* Clear received doorbell. */ 169 (*hw->ntb_ops->db_clear)(dev, 1); 170 171 /** 172 * Peer dev is already up. All mw settings are already done. 173 * Skip them. 174 */ 175 if (hw->peer_dev_up) 176 return; 177 178 if (hw->ntb_ops->spad_read == NULL) { 179 NTB_LOG(ERR, "Scratchpad read is not supported."); 180 return; 181 } 182 183 /* Check if mw setting on the peer is the same as local. */ 184 peer_mw_cnt = (*hw->ntb_ops->spad_read)(dev, SPAD_NUM_MWS, 0); 185 if (peer_mw_cnt != hw->mw_cnt) { 186 NTB_LOG(ERR, "Both mw cnt must be the same."); 187 return; 188 } 189 190 for (i = 0; i < hw->mw_cnt; i++) { 191 val_h = (*hw->ntb_ops->spad_read) 192 (dev, SPAD_MW0_SZ_H + 2 * i, 0); 193 val_l = (*hw->ntb_ops->spad_read) 194 (dev, SPAD_MW0_SZ_L + 2 * i, 0); 195 peer_mw_size = ((uint64_t)val_h << 32) | val_l; 196 NTB_LOG(DEBUG, "Peer %u mw size: 0x%"PRIx64"", i, 197 peer_mw_size); 198 if (peer_mw_size != hw->mw_size[i]) { 199 NTB_LOG(ERR, "Mw config must be the same."); 200 return; 201 } 202 } 203 204 hw->peer_dev_up = 1; 205 206 /** 207 * Handshake with peer. Spad_write & mw_set_trans only works 208 * when both devices are up. So write spad again when db is 209 * received. And set db again for the later device who may miss 210 * the 1st db. 211 */ 212 if (ntb_handshake_work(dev) < 0) { 213 NTB_LOG(ERR, "Handshake work failed."); 214 return; 215 } 216 217 /* To get the link info. */ 218 if (hw->ntb_ops->get_link_status == NULL) { 219 NTB_LOG(ERR, "Not supported to get link status."); 220 return; 221 } 222 (*hw->ntb_ops->get_link_status)(dev); 223 NTB_LOG(INFO, "Link is up. Link speed: %u. Link width: %u", 224 hw->link_speed, hw->link_width); 225 return; 226 } 227 228 if (db_bits & (1 << 1)) { 229 NTB_LOG(INFO, "DB1: Peer device is down."); 230 /* Clear received doorbell. */ 231 (*hw->ntb_ops->db_clear)(dev, 2); 232 233 /* Peer device will be down, So clean local side too. */ 234 ntb_link_cleanup(dev); 235 236 hw->peer_dev_up = 0; 237 /* Response peer's dev_stop request. */ 238 (*hw->ntb_ops->peer_db_set)(dev, 2); 239 return; 240 } 241 242 if (db_bits & (1 << 2)) { 243 NTB_LOG(INFO, "DB2: Peer device agrees dev to be down."); 244 /* Clear received doorbell. */ 245 (*hw->ntb_ops->db_clear)(dev, (1 << 2)); 246 hw->peer_dev_up = 0; 247 return; 248 } 249 } 250 251 static void 252 ntb_queue_conf_get(struct rte_rawdev *dev, 253 uint16_t queue_id, 254 rte_rawdev_obj_t queue_conf) 255 { 256 struct ntb_queue_conf *q_conf = queue_conf; 257 struct ntb_hw *hw = dev->dev_private; 258 259 q_conf->tx_free_thresh = hw->tx_queues[queue_id]->tx_free_thresh; 260 q_conf->nb_desc = hw->rx_queues[queue_id]->nb_rx_desc; 261 q_conf->rx_mp = hw->rx_queues[queue_id]->mpool; 262 } 263 264 static void 265 ntb_rxq_release_mbufs(struct ntb_rx_queue *q) 266 { 267 int i; 268 269 if (!q || !q->sw_ring) { 270 NTB_LOG(ERR, "Pointer to rxq or sw_ring is NULL"); 271 return; 272 } 273 274 for (i = 0; i < q->nb_rx_desc; i++) { 275 if (q->sw_ring[i].mbuf) { 276 rte_pktmbuf_free_seg(q->sw_ring[i].mbuf); 277 q->sw_ring[i].mbuf = NULL; 278 } 279 } 280 } 281 282 static void 283 ntb_rxq_release(struct ntb_rx_queue *rxq) 284 { 285 if (!rxq) { 286 NTB_LOG(ERR, "Pointer to rxq is NULL"); 287 return; 288 } 289 290 ntb_rxq_release_mbufs(rxq); 291 292 rte_free(rxq->sw_ring); 293 rte_free(rxq); 294 } 295 296 static int 297 ntb_rxq_setup(struct rte_rawdev *dev, 298 uint16_t qp_id, 299 rte_rawdev_obj_t queue_conf) 300 { 301 struct ntb_queue_conf *rxq_conf = queue_conf; 302 struct ntb_hw *hw = dev->dev_private; 303 struct ntb_rx_queue *rxq; 304 305 /* Allocate the rx queue data structure */ 306 rxq = rte_zmalloc_socket("ntb rx queue", 307 sizeof(struct ntb_rx_queue), 308 RTE_CACHE_LINE_SIZE, 309 dev->socket_id); 310 if (!rxq) { 311 NTB_LOG(ERR, "Failed to allocate memory for " 312 "rx queue data structure."); 313 return -ENOMEM; 314 } 315 316 if (rxq_conf->rx_mp == NULL) { 317 NTB_LOG(ERR, "Invalid null mempool pointer."); 318 return -EINVAL; 319 } 320 rxq->nb_rx_desc = rxq_conf->nb_desc; 321 rxq->mpool = rxq_conf->rx_mp; 322 rxq->port_id = dev->dev_id; 323 rxq->queue_id = qp_id; 324 rxq->hw = hw; 325 326 /* Allocate the software ring. */ 327 rxq->sw_ring = 328 rte_zmalloc_socket("ntb rx sw ring", 329 sizeof(struct ntb_rx_entry) * 330 rxq->nb_rx_desc, 331 RTE_CACHE_LINE_SIZE, 332 dev->socket_id); 333 if (!rxq->sw_ring) { 334 ntb_rxq_release(rxq); 335 rxq = NULL; 336 NTB_LOG(ERR, "Failed to allocate memory for SW ring"); 337 return -ENOMEM; 338 } 339 340 hw->rx_queues[qp_id] = rxq; 341 342 return 0; 343 } 344 345 static void 346 ntb_txq_release_mbufs(struct ntb_tx_queue *q) 347 { 348 int i; 349 350 if (!q || !q->sw_ring) { 351 NTB_LOG(ERR, "Pointer to txq or sw_ring is NULL"); 352 return; 353 } 354 355 for (i = 0; i < q->nb_tx_desc; i++) { 356 if (q->sw_ring[i].mbuf) { 357 rte_pktmbuf_free_seg(q->sw_ring[i].mbuf); 358 q->sw_ring[i].mbuf = NULL; 359 } 360 } 361 } 362 363 static void 364 ntb_txq_release(struct ntb_tx_queue *txq) 365 { 366 if (!txq) { 367 NTB_LOG(ERR, "Pointer to txq is NULL"); 368 return; 369 } 370 371 ntb_txq_release_mbufs(txq); 372 373 rte_free(txq->sw_ring); 374 rte_free(txq); 375 } 376 377 static int 378 ntb_txq_setup(struct rte_rawdev *dev, 379 uint16_t qp_id, 380 rte_rawdev_obj_t queue_conf) 381 { 382 struct ntb_queue_conf *txq_conf = queue_conf; 383 struct ntb_hw *hw = dev->dev_private; 384 struct ntb_tx_queue *txq; 385 uint16_t i, prev; 386 387 /* Allocate the TX queue data structure. */ 388 txq = rte_zmalloc_socket("ntb tx queue", 389 sizeof(struct ntb_tx_queue), 390 RTE_CACHE_LINE_SIZE, 391 dev->socket_id); 392 if (!txq) { 393 NTB_LOG(ERR, "Failed to allocate memory for " 394 "tx queue structure"); 395 return -ENOMEM; 396 } 397 398 txq->nb_tx_desc = txq_conf->nb_desc; 399 txq->port_id = dev->dev_id; 400 txq->queue_id = qp_id; 401 txq->hw = hw; 402 403 /* Allocate software ring */ 404 txq->sw_ring = 405 rte_zmalloc_socket("ntb tx sw ring", 406 sizeof(struct ntb_tx_entry) * 407 txq->nb_tx_desc, 408 RTE_CACHE_LINE_SIZE, 409 dev->socket_id); 410 if (!txq->sw_ring) { 411 ntb_txq_release(txq); 412 txq = NULL; 413 NTB_LOG(ERR, "Failed to allocate memory for SW TX ring"); 414 return -ENOMEM; 415 } 416 417 prev = txq->nb_tx_desc - 1; 418 for (i = 0; i < txq->nb_tx_desc; i++) { 419 txq->sw_ring[i].mbuf = NULL; 420 txq->sw_ring[i].last_id = i; 421 txq->sw_ring[prev].next_id = i; 422 prev = i; 423 } 424 425 txq->tx_free_thresh = txq_conf->tx_free_thresh ? 426 txq_conf->tx_free_thresh : 427 NTB_DFLT_TX_FREE_THRESH; 428 if (txq->tx_free_thresh >= txq->nb_tx_desc - 3) { 429 NTB_LOG(ERR, "tx_free_thresh must be less than nb_desc - 3. " 430 "(tx_free_thresh=%u qp_id=%u)", txq->tx_free_thresh, 431 qp_id); 432 return -EINVAL; 433 } 434 435 hw->tx_queues[qp_id] = txq; 436 437 return 0; 438 } 439 440 441 static int 442 ntb_queue_setup(struct rte_rawdev *dev, 443 uint16_t queue_id, 444 rte_rawdev_obj_t queue_conf) 445 { 446 struct ntb_hw *hw = dev->dev_private; 447 int ret; 448 449 if (queue_id >= hw->queue_pairs) 450 return -EINVAL; 451 452 ret = ntb_txq_setup(dev, queue_id, queue_conf); 453 if (ret < 0) 454 return ret; 455 456 ret = ntb_rxq_setup(dev, queue_id, queue_conf); 457 458 return ret; 459 } 460 461 static int 462 ntb_queue_release(struct rte_rawdev *dev, uint16_t queue_id) 463 { 464 struct ntb_hw *hw = dev->dev_private; 465 466 if (queue_id >= hw->queue_pairs) 467 return -EINVAL; 468 469 ntb_txq_release(hw->tx_queues[queue_id]); 470 hw->tx_queues[queue_id] = NULL; 471 ntb_rxq_release(hw->rx_queues[queue_id]); 472 hw->rx_queues[queue_id] = NULL; 473 474 return 0; 475 } 476 477 static uint16_t 478 ntb_queue_count(struct rte_rawdev *dev) 479 { 480 struct ntb_hw *hw = dev->dev_private; 481 return hw->queue_pairs; 482 } 483 484 static int 485 ntb_queue_init(struct rte_rawdev *dev, uint16_t qp_id) 486 { 487 struct ntb_hw *hw = dev->dev_private; 488 struct ntb_rx_queue *rxq = hw->rx_queues[qp_id]; 489 struct ntb_tx_queue *txq = hw->tx_queues[qp_id]; 490 volatile struct ntb_header *local_hdr; 491 struct ntb_header *remote_hdr; 492 uint16_t q_size = hw->queue_size; 493 uint32_t hdr_offset; 494 void *bar_addr; 495 uint16_t i; 496 497 if (hw->ntb_ops->get_peer_mw_addr == NULL) { 498 NTB_LOG(ERR, "Getting peer mw addr is not supported."); 499 return -EINVAL; 500 } 501 502 /* Put queue info into the start of shared memory. */ 503 hdr_offset = hw->hdr_size_per_queue * qp_id; 504 local_hdr = (volatile struct ntb_header *) 505 ((size_t)hw->mz[0]->addr + hdr_offset); 506 bar_addr = (*hw->ntb_ops->get_peer_mw_addr)(dev, 0); 507 if (bar_addr == NULL) 508 return -EINVAL; 509 remote_hdr = (struct ntb_header *) 510 ((size_t)bar_addr + hdr_offset); 511 512 /* rxq init. */ 513 rxq->rx_desc_ring = (struct ntb_desc *) 514 (&remote_hdr->desc_ring); 515 rxq->rx_used_ring = (volatile struct ntb_used *) 516 (&local_hdr->desc_ring[q_size]); 517 rxq->avail_cnt = &remote_hdr->avail_cnt; 518 rxq->used_cnt = &local_hdr->used_cnt; 519 520 for (i = 0; i < rxq->nb_rx_desc - 1; i++) { 521 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mpool); 522 if (unlikely(!mbuf)) { 523 NTB_LOG(ERR, "Failed to allocate mbuf for RX"); 524 return -ENOMEM; 525 } 526 mbuf->port = dev->dev_id; 527 528 rxq->sw_ring[i].mbuf = mbuf; 529 530 rxq->rx_desc_ring[i].addr = rte_pktmbuf_mtod(mbuf, size_t); 531 rxq->rx_desc_ring[i].len = mbuf->buf_len - RTE_PKTMBUF_HEADROOM; 532 } 533 rte_wmb(); 534 *rxq->avail_cnt = rxq->nb_rx_desc - 1; 535 rxq->last_avail = rxq->nb_rx_desc - 1; 536 rxq->last_used = 0; 537 538 /* txq init */ 539 txq->tx_desc_ring = (volatile struct ntb_desc *) 540 (&local_hdr->desc_ring); 541 txq->tx_used_ring = (struct ntb_used *) 542 (&remote_hdr->desc_ring[q_size]); 543 txq->avail_cnt = &local_hdr->avail_cnt; 544 txq->used_cnt = &remote_hdr->used_cnt; 545 546 rte_wmb(); 547 *txq->used_cnt = 0; 548 txq->last_used = 0; 549 txq->last_avail = 0; 550 txq->nb_tx_free = txq->nb_tx_desc - 1; 551 552 /* Set per queue stats. */ 553 for (i = 0; i < NTB_XSTATS_NUM; i++) { 554 hw->ntb_xstats[i + NTB_XSTATS_NUM * (qp_id + 1)] = 0; 555 hw->ntb_xstats_off[i + NTB_XSTATS_NUM * (qp_id + 1)] = 0; 556 } 557 558 return 0; 559 } 560 561 static inline void 562 ntb_enqueue_cleanup(struct ntb_tx_queue *txq) 563 { 564 struct ntb_tx_entry *sw_ring = txq->sw_ring; 565 uint16_t tx_free = txq->last_avail; 566 uint16_t nb_to_clean, i; 567 568 /* avail_cnt + 1 represents where to rx next in the peer. */ 569 nb_to_clean = (*txq->avail_cnt - txq->last_avail + 1 + 570 txq->nb_tx_desc) & (txq->nb_tx_desc - 1); 571 nb_to_clean = RTE_MIN(nb_to_clean, txq->tx_free_thresh); 572 for (i = 0; i < nb_to_clean; i++) { 573 if (sw_ring[tx_free].mbuf) 574 rte_pktmbuf_free_seg(sw_ring[tx_free].mbuf); 575 tx_free = (tx_free + 1) & (txq->nb_tx_desc - 1); 576 } 577 578 txq->nb_tx_free += nb_to_clean; 579 txq->last_avail = tx_free; 580 } 581 582 static int 583 ntb_enqueue_bufs(struct rte_rawdev *dev, 584 struct rte_rawdev_buf **buffers, 585 unsigned int count, 586 rte_rawdev_obj_t context) 587 { 588 struct ntb_hw *hw = dev->dev_private; 589 struct ntb_tx_queue *txq = hw->tx_queues[(size_t)context]; 590 struct ntb_tx_entry *sw_ring = txq->sw_ring; 591 struct rte_mbuf *txm; 592 struct ntb_used tx_used[NTB_MAX_DESC_SIZE]; 593 volatile struct ntb_desc *tx_item; 594 uint16_t tx_last, nb_segs, off, last_used, avail_cnt; 595 uint16_t nb_mbufs = 0; 596 uint16_t nb_tx = 0; 597 uint64_t bytes = 0; 598 void *buf_addr; 599 int i; 600 601 if (unlikely(hw->ntb_ops->ioremap == NULL)) { 602 NTB_LOG(ERR, "Ioremap not supported."); 603 return nb_tx; 604 } 605 606 if (unlikely(dev->started == 0 || hw->peer_dev_up == 0)) { 607 NTB_LOG(DEBUG, "Link is not up."); 608 return nb_tx; 609 } 610 611 if (txq->nb_tx_free < txq->tx_free_thresh) 612 ntb_enqueue_cleanup(txq); 613 614 off = NTB_XSTATS_NUM * ((size_t)context + 1); 615 last_used = txq->last_used; 616 avail_cnt = *txq->avail_cnt;/* Where to alloc next. */ 617 for (nb_tx = 0; nb_tx < count; nb_tx++) { 618 txm = (struct rte_mbuf *)(buffers[nb_tx]->buf_addr); 619 if (txm == NULL || txq->nb_tx_free < txm->nb_segs) 620 break; 621 622 tx_last = (txq->last_used + txm->nb_segs - 1) & 623 (txq->nb_tx_desc - 1); 624 nb_segs = txm->nb_segs; 625 for (i = 0; i < nb_segs; i++) { 626 /* Not enough ring space for tx. */ 627 if (txq->last_used == avail_cnt) 628 goto end_of_tx; 629 sw_ring[txq->last_used].mbuf = txm; 630 tx_item = txq->tx_desc_ring + txq->last_used; 631 632 if (!tx_item->len) { 633 (hw->ntb_xstats[NTB_TX_ERRS_ID + off])++; 634 goto end_of_tx; 635 } 636 if (txm->data_len > tx_item->len) { 637 NTB_LOG(ERR, "Data length exceeds buf length." 638 " Only %u data would be transmitted.", 639 tx_item->len); 640 txm->data_len = tx_item->len; 641 } 642 643 /* translate remote virtual addr to bar virtual addr */ 644 buf_addr = (*hw->ntb_ops->ioremap)(dev, tx_item->addr); 645 if (buf_addr == NULL) { 646 (hw->ntb_xstats[NTB_TX_ERRS_ID + off])++; 647 NTB_LOG(ERR, "Null remap addr."); 648 goto end_of_tx; 649 } 650 rte_memcpy(buf_addr, rte_pktmbuf_mtod(txm, void *), 651 txm->data_len); 652 653 tx_used[nb_mbufs].len = txm->data_len; 654 tx_used[nb_mbufs++].flags = (txq->last_used == 655 tx_last) ? 656 NTB_FLAG_EOP : 0; 657 658 /* update stats */ 659 bytes += txm->data_len; 660 661 txm = txm->next; 662 663 sw_ring[txq->last_used].next_id = (txq->last_used + 1) & 664 (txq->nb_tx_desc - 1); 665 sw_ring[txq->last_used].last_id = tx_last; 666 txq->last_used = (txq->last_used + 1) & 667 (txq->nb_tx_desc - 1); 668 } 669 txq->nb_tx_free -= nb_segs; 670 } 671 672 end_of_tx: 673 if (nb_tx) { 674 uint16_t nb1, nb2; 675 if (nb_mbufs > txq->nb_tx_desc - last_used) { 676 nb1 = txq->nb_tx_desc - last_used; 677 nb2 = nb_mbufs - txq->nb_tx_desc + last_used; 678 } else { 679 nb1 = nb_mbufs; 680 nb2 = 0; 681 } 682 rte_memcpy(txq->tx_used_ring + last_used, tx_used, 683 sizeof(struct ntb_used) * nb1); 684 rte_memcpy(txq->tx_used_ring, tx_used + nb1, 685 sizeof(struct ntb_used) * nb2); 686 *txq->used_cnt = txq->last_used; 687 rte_wmb(); 688 689 /* update queue stats */ 690 hw->ntb_xstats[NTB_TX_BYTES_ID + off] += bytes; 691 hw->ntb_xstats[NTB_TX_PKTS_ID + off] += nb_tx; 692 } 693 694 return nb_tx; 695 } 696 697 static int 698 ntb_dequeue_bufs(struct rte_rawdev *dev, 699 struct rte_rawdev_buf **buffers, 700 unsigned int count, 701 rte_rawdev_obj_t context) 702 { 703 struct ntb_hw *hw = dev->dev_private; 704 struct ntb_rx_queue *rxq = hw->rx_queues[(size_t)context]; 705 struct ntb_rx_entry *sw_ring = rxq->sw_ring; 706 struct ntb_desc rx_desc[NTB_MAX_DESC_SIZE]; 707 struct rte_mbuf *first, *rxm_t; 708 struct rte_mbuf *prev = NULL; 709 volatile struct ntb_used *rx_item; 710 uint16_t nb_mbufs = 0; 711 uint16_t nb_rx = 0; 712 uint64_t bytes = 0; 713 uint16_t off, last_avail, used_cnt, used_nb; 714 int i; 715 716 if (unlikely(dev->started == 0 || hw->peer_dev_up == 0)) { 717 NTB_LOG(DEBUG, "Link is not up"); 718 return nb_rx; 719 } 720 721 used_cnt = *rxq->used_cnt; 722 723 if (rxq->last_used == used_cnt) 724 return nb_rx; 725 726 last_avail = rxq->last_avail; 727 used_nb = (used_cnt - rxq->last_used) & (rxq->nb_rx_desc - 1); 728 count = RTE_MIN(count, used_nb); 729 for (nb_rx = 0; nb_rx < count; nb_rx++) { 730 i = 0; 731 while (true) { 732 rx_item = rxq->rx_used_ring + rxq->last_used; 733 rxm_t = sw_ring[rxq->last_used].mbuf; 734 rxm_t->data_len = rx_item->len; 735 rxm_t->data_off = RTE_PKTMBUF_HEADROOM; 736 rxm_t->port = rxq->port_id; 737 738 if (!i) { 739 rxm_t->nb_segs = 1; 740 first = rxm_t; 741 first->pkt_len = 0; 742 buffers[nb_rx]->buf_addr = rxm_t; 743 } else { 744 prev->next = rxm_t; 745 first->nb_segs++; 746 } 747 748 prev = rxm_t; 749 first->pkt_len += prev->data_len; 750 rxq->last_used = (rxq->last_used + 1) & 751 (rxq->nb_rx_desc - 1); 752 753 /* alloc new mbuf */ 754 rxm_t = rte_mbuf_raw_alloc(rxq->mpool); 755 if (unlikely(rxm_t == NULL)) { 756 NTB_LOG(ERR, "recv alloc mbuf failed."); 757 goto end_of_rx; 758 } 759 rxm_t->port = rxq->port_id; 760 sw_ring[rxq->last_avail].mbuf = rxm_t; 761 i++; 762 763 /* fill new desc */ 764 rx_desc[nb_mbufs].addr = 765 rte_pktmbuf_mtod(rxm_t, size_t); 766 rx_desc[nb_mbufs++].len = rxm_t->buf_len - 767 RTE_PKTMBUF_HEADROOM; 768 rxq->last_avail = (rxq->last_avail + 1) & 769 (rxq->nb_rx_desc - 1); 770 771 if (rx_item->flags & NTB_FLAG_EOP) 772 break; 773 } 774 /* update stats */ 775 bytes += first->pkt_len; 776 } 777 778 end_of_rx: 779 if (nb_rx) { 780 uint16_t nb1, nb2; 781 if (nb_mbufs > rxq->nb_rx_desc - last_avail) { 782 nb1 = rxq->nb_rx_desc - last_avail; 783 nb2 = nb_mbufs - rxq->nb_rx_desc + last_avail; 784 } else { 785 nb1 = nb_mbufs; 786 nb2 = 0; 787 } 788 rte_memcpy(rxq->rx_desc_ring + last_avail, rx_desc, 789 sizeof(struct ntb_desc) * nb1); 790 rte_memcpy(rxq->rx_desc_ring, rx_desc + nb1, 791 sizeof(struct ntb_desc) * nb2); 792 *rxq->avail_cnt = rxq->last_avail; 793 rte_wmb(); 794 795 /* update queue stats */ 796 off = NTB_XSTATS_NUM * ((size_t)context + 1); 797 hw->ntb_xstats[NTB_RX_BYTES_ID + off] += bytes; 798 hw->ntb_xstats[NTB_RX_PKTS_ID + off] += nb_rx; 799 hw->ntb_xstats[NTB_RX_MISS_ID + off] += (count - nb_rx); 800 } 801 802 return nb_rx; 803 } 804 805 static void 806 ntb_dev_info_get(struct rte_rawdev *dev, rte_rawdev_obj_t dev_info) 807 { 808 struct ntb_hw *hw = dev->dev_private; 809 struct ntb_dev_info *info = dev_info; 810 811 info->mw_cnt = hw->mw_cnt; 812 info->mw_size = hw->mw_size; 813 814 /** 815 * Intel hardware requires that mapped memory base address should be 816 * aligned with EMBARSZ and needs continuous memzone. 817 */ 818 info->mw_size_align = (uint8_t)(hw->pci_dev->id.vendor_id == 819 NTB_INTEL_VENDOR_ID); 820 821 if (!hw->queue_size || !hw->queue_pairs) { 822 NTB_LOG(ERR, "No queue size and queue num assigned."); 823 return; 824 } 825 826 hw->hdr_size_per_queue = RTE_ALIGN(sizeof(struct ntb_header) + 827 hw->queue_size * sizeof(struct ntb_desc) + 828 hw->queue_size * sizeof(struct ntb_used), 829 RTE_CACHE_LINE_SIZE); 830 info->ntb_hdr_size = hw->hdr_size_per_queue * hw->queue_pairs; 831 } 832 833 static int 834 ntb_dev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config) 835 { 836 struct ntb_dev_config *conf = config; 837 struct ntb_hw *hw = dev->dev_private; 838 uint32_t xstats_num; 839 int ret; 840 841 hw->queue_pairs = conf->num_queues; 842 hw->queue_size = conf->queue_size; 843 hw->used_mw_num = conf->mz_num; 844 hw->mz = conf->mz_list; 845 hw->rx_queues = rte_zmalloc("ntb_rx_queues", 846 sizeof(struct ntb_rx_queue *) * hw->queue_pairs, 0); 847 hw->tx_queues = rte_zmalloc("ntb_tx_queues", 848 sizeof(struct ntb_tx_queue *) * hw->queue_pairs, 0); 849 /* First total stats, then per queue stats. */ 850 xstats_num = (hw->queue_pairs + 1) * NTB_XSTATS_NUM; 851 hw->ntb_xstats = rte_zmalloc("ntb_xstats", xstats_num * 852 sizeof(uint64_t), 0); 853 hw->ntb_xstats_off = rte_zmalloc("ntb_xstats_off", xstats_num * 854 sizeof(uint64_t), 0); 855 856 /* Start handshake with the peer. */ 857 ret = ntb_handshake_work(dev); 858 if (ret < 0) { 859 rte_free(hw->rx_queues); 860 rte_free(hw->tx_queues); 861 hw->rx_queues = NULL; 862 hw->tx_queues = NULL; 863 return ret; 864 } 865 866 return 0; 867 } 868 869 static int 870 ntb_dev_start(struct rte_rawdev *dev) 871 { 872 struct ntb_hw *hw = dev->dev_private; 873 uint32_t peer_base_l, peer_val; 874 uint64_t peer_base_h; 875 uint32_t i; 876 int ret; 877 878 if (!hw->link_status || !hw->peer_dev_up) 879 return -EINVAL; 880 881 /* Set total stats. */ 882 for (i = 0; i < NTB_XSTATS_NUM; i++) { 883 hw->ntb_xstats[i] = 0; 884 hw->ntb_xstats_off[i] = 0; 885 } 886 887 for (i = 0; i < hw->queue_pairs; i++) { 888 ret = ntb_queue_init(dev, i); 889 if (ret) { 890 NTB_LOG(ERR, "Failed to init queue."); 891 goto err_q_init; 892 } 893 } 894 895 hw->peer_mw_base = rte_zmalloc("ntb_peer_mw_base", hw->mw_cnt * 896 sizeof(uint64_t), 0); 897 898 if (hw->ntb_ops->spad_read == NULL) { 899 ret = -ENOTSUP; 900 goto err_up; 901 } 902 903 peer_val = (*hw->ntb_ops->spad_read)(dev, SPAD_Q_SZ, 0); 904 if (peer_val != hw->queue_size) { 905 NTB_LOG(ERR, "Inconsistent queue size! (local: %u peer: %u)", 906 hw->queue_size, peer_val); 907 ret = -EINVAL; 908 goto err_up; 909 } 910 911 peer_val = (*hw->ntb_ops->spad_read)(dev, SPAD_NUM_QPS, 0); 912 if (peer_val != hw->queue_pairs) { 913 NTB_LOG(ERR, "Inconsistent number of queues! (local: %u peer:" 914 " %u)", hw->queue_pairs, peer_val); 915 ret = -EINVAL; 916 goto err_up; 917 } 918 919 hw->peer_used_mws = (*hw->ntb_ops->spad_read)(dev, SPAD_USED_MWS, 0); 920 921 for (i = 0; i < hw->peer_used_mws; i++) { 922 peer_base_h = (*hw->ntb_ops->spad_read)(dev, 923 SPAD_MW0_BA_H + 2 * i, 0); 924 peer_base_l = (*hw->ntb_ops->spad_read)(dev, 925 SPAD_MW0_BA_L + 2 * i, 0); 926 hw->peer_mw_base[i] = (peer_base_h << 32) + peer_base_l; 927 } 928 929 dev->started = 1; 930 931 return 0; 932 933 err_up: 934 rte_free(hw->peer_mw_base); 935 err_q_init: 936 for (i = 0; i < hw->queue_pairs; i++) { 937 ntb_rxq_release_mbufs(hw->rx_queues[i]); 938 ntb_txq_release_mbufs(hw->tx_queues[i]); 939 } 940 941 return ret; 942 } 943 944 static void 945 ntb_dev_stop(struct rte_rawdev *dev) 946 { 947 struct ntb_hw *hw = dev->dev_private; 948 uint32_t time_out; 949 int status, i; 950 951 if (!hw->peer_dev_up) 952 goto clean; 953 954 ntb_link_cleanup(dev); 955 956 /* Notify the peer that device will be down. */ 957 if (hw->ntb_ops->peer_db_set == NULL) { 958 NTB_LOG(ERR, "Peer doorbell setting is not supported."); 959 return; 960 } 961 status = (*hw->ntb_ops->peer_db_set)(dev, 1); 962 if (status) { 963 NTB_LOG(ERR, "Failed to tell peer device is down."); 964 return; 965 } 966 967 /* 968 * Set time out as 1s in case that the peer is stopped accidently 969 * without any notification. 970 */ 971 time_out = 1000000; 972 973 /* Wait for cleanup work down before db mask clear. */ 974 while (hw->peer_dev_up && time_out) { 975 time_out -= 10; 976 rte_delay_us(10); 977 } 978 979 clean: 980 /* Clear doorbells mask. */ 981 if (hw->ntb_ops->db_set_mask == NULL) { 982 NTB_LOG(ERR, "Doorbell mask setting is not supported."); 983 return; 984 } 985 status = (*hw->ntb_ops->db_set_mask)(dev, 986 (((uint64_t)1 << hw->db_cnt) - 1)); 987 if (status) 988 NTB_LOG(ERR, "Failed to clear doorbells."); 989 990 for (i = 0; i < hw->queue_pairs; i++) { 991 ntb_rxq_release_mbufs(hw->rx_queues[i]); 992 ntb_txq_release_mbufs(hw->tx_queues[i]); 993 } 994 995 dev->started = 0; 996 } 997 998 static int 999 ntb_dev_close(struct rte_rawdev *dev) 1000 { 1001 struct ntb_hw *hw = dev->dev_private; 1002 struct rte_intr_handle *intr_handle; 1003 int i; 1004 1005 if (dev->started) 1006 ntb_dev_stop(dev); 1007 1008 /* free queues */ 1009 for (i = 0; i < hw->queue_pairs; i++) 1010 ntb_queue_release(dev, i); 1011 hw->queue_pairs = 0; 1012 1013 intr_handle = &hw->pci_dev->intr_handle; 1014 /* Clean datapath event and vec mapping */ 1015 rte_intr_efd_disable(intr_handle); 1016 if (intr_handle->intr_vec) { 1017 rte_free(intr_handle->intr_vec); 1018 intr_handle->intr_vec = NULL; 1019 } 1020 /* Disable uio intr before callback unregister */ 1021 rte_intr_disable(intr_handle); 1022 1023 /* Unregister callback func to eal lib */ 1024 rte_intr_callback_unregister(intr_handle, 1025 ntb_dev_intr_handler, dev); 1026 1027 return 0; 1028 } 1029 1030 static int 1031 ntb_dev_reset(struct rte_rawdev *rawdev __rte_unused) 1032 { 1033 return 0; 1034 } 1035 1036 static int 1037 ntb_attr_set(struct rte_rawdev *dev, const char *attr_name, 1038 uint64_t attr_value) 1039 { 1040 struct ntb_hw *hw; 1041 int index; 1042 1043 if (dev == NULL || attr_name == NULL) { 1044 NTB_LOG(ERR, "Invalid arguments for setting attributes"); 1045 return -EINVAL; 1046 } 1047 1048 hw = dev->dev_private; 1049 1050 if (!strncmp(attr_name, NTB_SPAD_USER, NTB_SPAD_USER_LEN)) { 1051 if (hw->ntb_ops->spad_write == NULL) 1052 return -ENOTSUP; 1053 index = atoi(&attr_name[NTB_SPAD_USER_LEN]); 1054 (*hw->ntb_ops->spad_write)(dev, hw->spad_user_list[index], 1055 1, attr_value); 1056 NTB_LOG(DEBUG, "Set attribute (%s) Value (%" PRIu64 ")", 1057 attr_name, attr_value); 1058 return 0; 1059 } 1060 1061 if (!strncmp(attr_name, NTB_QUEUE_SZ_NAME, NTB_ATTR_NAME_LEN)) { 1062 hw->queue_size = attr_value; 1063 NTB_LOG(DEBUG, "Set attribute (%s) Value (%" PRIu64 ")", 1064 attr_name, attr_value); 1065 return 0; 1066 } 1067 1068 if (!strncmp(attr_name, NTB_QUEUE_NUM_NAME, NTB_ATTR_NAME_LEN)) { 1069 hw->queue_pairs = attr_value; 1070 NTB_LOG(DEBUG, "Set attribute (%s) Value (%" PRIu64 ")", 1071 attr_name, attr_value); 1072 return 0; 1073 } 1074 1075 /* Attribute not found. */ 1076 NTB_LOG(ERR, "Attribute not found."); 1077 return -EINVAL; 1078 } 1079 1080 static int 1081 ntb_attr_get(struct rte_rawdev *dev, const char *attr_name, 1082 uint64_t *attr_value) 1083 { 1084 struct ntb_hw *hw; 1085 int index; 1086 1087 if (dev == NULL || attr_name == NULL || attr_value == NULL) { 1088 NTB_LOG(ERR, "Invalid arguments for getting attributes"); 1089 return -EINVAL; 1090 } 1091 1092 hw = dev->dev_private; 1093 1094 if (!strncmp(attr_name, NTB_TOPO_NAME, NTB_ATTR_NAME_LEN)) { 1095 *attr_value = hw->topo; 1096 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")", 1097 attr_name, *attr_value); 1098 return 0; 1099 } 1100 1101 if (!strncmp(attr_name, NTB_LINK_STATUS_NAME, NTB_ATTR_NAME_LEN)) { 1102 /* hw->link_status only indicates hw link status. */ 1103 *attr_value = hw->link_status && hw->peer_dev_up; 1104 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")", 1105 attr_name, *attr_value); 1106 return 0; 1107 } 1108 1109 if (!strncmp(attr_name, NTB_SPEED_NAME, NTB_ATTR_NAME_LEN)) { 1110 *attr_value = hw->link_speed; 1111 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")", 1112 attr_name, *attr_value); 1113 return 0; 1114 } 1115 1116 if (!strncmp(attr_name, NTB_WIDTH_NAME, NTB_ATTR_NAME_LEN)) { 1117 *attr_value = hw->link_width; 1118 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")", 1119 attr_name, *attr_value); 1120 return 0; 1121 } 1122 1123 if (!strncmp(attr_name, NTB_MW_CNT_NAME, NTB_ATTR_NAME_LEN)) { 1124 *attr_value = hw->mw_cnt; 1125 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")", 1126 attr_name, *attr_value); 1127 return 0; 1128 } 1129 1130 if (!strncmp(attr_name, NTB_DB_CNT_NAME, NTB_ATTR_NAME_LEN)) { 1131 *attr_value = hw->db_cnt; 1132 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")", 1133 attr_name, *attr_value); 1134 return 0; 1135 } 1136 1137 if (!strncmp(attr_name, NTB_SPAD_CNT_NAME, NTB_ATTR_NAME_LEN)) { 1138 *attr_value = hw->spad_cnt; 1139 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")", 1140 attr_name, *attr_value); 1141 return 0; 1142 } 1143 1144 if (!strncmp(attr_name, NTB_SPAD_USER, NTB_SPAD_USER_LEN)) { 1145 if (hw->ntb_ops->spad_read == NULL) 1146 return -ENOTSUP; 1147 index = atoi(&attr_name[NTB_SPAD_USER_LEN]); 1148 *attr_value = (*hw->ntb_ops->spad_read)(dev, 1149 hw->spad_user_list[index], 0); 1150 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")", 1151 attr_name, *attr_value); 1152 return 0; 1153 } 1154 1155 /* Attribute not found. */ 1156 NTB_LOG(ERR, "Attribute not found."); 1157 return -EINVAL; 1158 } 1159 1160 static inline uint64_t 1161 ntb_stats_update(uint64_t offset, uint64_t stat) 1162 { 1163 if (stat >= offset) 1164 return (stat - offset); 1165 else 1166 return (uint64_t)(((uint64_t)-1) - offset + stat + 1); 1167 } 1168 1169 static int 1170 ntb_xstats_get(const struct rte_rawdev *dev, 1171 const unsigned int ids[], 1172 uint64_t values[], 1173 unsigned int n) 1174 { 1175 struct ntb_hw *hw = dev->dev_private; 1176 uint32_t i, j, off, xstats_num; 1177 1178 /* Calculate total stats of all queues. */ 1179 for (i = 0; i < NTB_XSTATS_NUM; i++) { 1180 hw->ntb_xstats[i] = 0; 1181 for (j = 0; j < hw->queue_pairs; j++) { 1182 off = NTB_XSTATS_NUM * (j + 1) + i; 1183 hw->ntb_xstats[i] += 1184 ntb_stats_update(hw->ntb_xstats_off[off], 1185 hw->ntb_xstats[off]); 1186 } 1187 } 1188 1189 xstats_num = NTB_XSTATS_NUM * (hw->queue_pairs + 1); 1190 for (i = 0; i < n && ids[i] < xstats_num; i++) { 1191 if (ids[i] < NTB_XSTATS_NUM) 1192 values[i] = hw->ntb_xstats[ids[i]]; 1193 else 1194 values[i] = 1195 ntb_stats_update(hw->ntb_xstats_off[ids[i]], 1196 hw->ntb_xstats[ids[i]]); 1197 } 1198 1199 return i; 1200 } 1201 1202 static int 1203 ntb_xstats_get_names(const struct rte_rawdev *dev, 1204 struct rte_rawdev_xstats_name *xstats_names, 1205 unsigned int size) 1206 { 1207 struct ntb_hw *hw = dev->dev_private; 1208 uint32_t xstats_num, i, j, off; 1209 1210 xstats_num = NTB_XSTATS_NUM * (hw->queue_pairs + 1); 1211 if (xstats_names == NULL || size < xstats_num) 1212 return xstats_num; 1213 1214 /* Total stats names */ 1215 memcpy(xstats_names, ntb_xstats_names, sizeof(ntb_xstats_names)); 1216 1217 /* Queue stats names */ 1218 for (i = 0; i < hw->queue_pairs; i++) { 1219 for (j = 0; j < NTB_XSTATS_NUM; j++) { 1220 off = j + (i + 1) * NTB_XSTATS_NUM; 1221 snprintf(xstats_names[off].name, 1222 sizeof(xstats_names[0].name), 1223 "%s_q%u", ntb_xstats_names[j].name, i); 1224 } 1225 } 1226 1227 return xstats_num; 1228 } 1229 1230 static uint64_t 1231 ntb_xstats_get_by_name(const struct rte_rawdev *dev, 1232 const char *name, unsigned int *id) 1233 { 1234 struct rte_rawdev_xstats_name *xstats_names; 1235 struct ntb_hw *hw = dev->dev_private; 1236 uint32_t xstats_num, i, j, off; 1237 1238 if (name == NULL) 1239 return -EINVAL; 1240 1241 xstats_num = NTB_XSTATS_NUM * (hw->queue_pairs + 1); 1242 xstats_names = rte_zmalloc("ntb_stats_name", 1243 sizeof(struct rte_rawdev_xstats_name) * 1244 xstats_num, 0); 1245 ntb_xstats_get_names(dev, xstats_names, xstats_num); 1246 1247 /* Calculate total stats of all queues. */ 1248 for (i = 0; i < NTB_XSTATS_NUM; i++) { 1249 for (j = 0; j < hw->queue_pairs; j++) { 1250 off = NTB_XSTATS_NUM * (j + 1) + i; 1251 hw->ntb_xstats[i] += 1252 ntb_stats_update(hw->ntb_xstats_off[off], 1253 hw->ntb_xstats[off]); 1254 } 1255 } 1256 1257 for (i = 0; i < xstats_num; i++) { 1258 if (!strncmp(name, xstats_names[i].name, 1259 RTE_RAW_DEV_XSTATS_NAME_SIZE)) { 1260 *id = i; 1261 rte_free(xstats_names); 1262 if (i < NTB_XSTATS_NUM) 1263 return hw->ntb_xstats[i]; 1264 else 1265 return ntb_stats_update(hw->ntb_xstats_off[i], 1266 hw->ntb_xstats[i]); 1267 } 1268 } 1269 1270 NTB_LOG(ERR, "Cannot find the xstats name."); 1271 1272 return -EINVAL; 1273 } 1274 1275 static int 1276 ntb_xstats_reset(struct rte_rawdev *dev, 1277 const uint32_t ids[], 1278 uint32_t nb_ids) 1279 { 1280 struct ntb_hw *hw = dev->dev_private; 1281 uint32_t i, j, off, xstats_num; 1282 1283 xstats_num = NTB_XSTATS_NUM * (hw->queue_pairs + 1); 1284 for (i = 0; i < nb_ids && ids[i] < xstats_num; i++) { 1285 if (ids[i] < NTB_XSTATS_NUM) { 1286 for (j = 0; j < hw->queue_pairs; j++) { 1287 off = NTB_XSTATS_NUM * (j + 1) + ids[i]; 1288 hw->ntb_xstats_off[off] = hw->ntb_xstats[off]; 1289 } 1290 } else { 1291 hw->ntb_xstats_off[ids[i]] = hw->ntb_xstats[ids[i]]; 1292 } 1293 } 1294 1295 return i; 1296 } 1297 1298 static const struct rte_rawdev_ops ntb_ops = { 1299 .dev_info_get = ntb_dev_info_get, 1300 .dev_configure = ntb_dev_configure, 1301 .dev_start = ntb_dev_start, 1302 .dev_stop = ntb_dev_stop, 1303 .dev_close = ntb_dev_close, 1304 .dev_reset = ntb_dev_reset, 1305 1306 .queue_def_conf = ntb_queue_conf_get, 1307 .queue_setup = ntb_queue_setup, 1308 .queue_release = ntb_queue_release, 1309 .queue_count = ntb_queue_count, 1310 1311 .enqueue_bufs = ntb_enqueue_bufs, 1312 .dequeue_bufs = ntb_dequeue_bufs, 1313 1314 .attr_get = ntb_attr_get, 1315 .attr_set = ntb_attr_set, 1316 1317 .xstats_get = ntb_xstats_get, 1318 .xstats_get_names = ntb_xstats_get_names, 1319 .xstats_get_by_name = ntb_xstats_get_by_name, 1320 .xstats_reset = ntb_xstats_reset, 1321 }; 1322 1323 static int 1324 ntb_init_hw(struct rte_rawdev *dev, struct rte_pci_device *pci_dev) 1325 { 1326 struct ntb_hw *hw = dev->dev_private; 1327 struct rte_intr_handle *intr_handle; 1328 int ret, i; 1329 1330 hw->pci_dev = pci_dev; 1331 hw->peer_dev_up = 0; 1332 hw->link_status = NTB_LINK_DOWN; 1333 hw->link_speed = NTB_SPEED_NONE; 1334 hw->link_width = NTB_WIDTH_NONE; 1335 1336 switch (pci_dev->id.device_id) { 1337 case NTB_INTEL_DEV_ID_B2B_SKX: 1338 hw->ntb_ops = &intel_ntb_ops; 1339 break; 1340 default: 1341 NTB_LOG(ERR, "Not supported device."); 1342 return -EINVAL; 1343 } 1344 1345 if (hw->ntb_ops->ntb_dev_init == NULL) 1346 return -ENOTSUP; 1347 ret = (*hw->ntb_ops->ntb_dev_init)(dev); 1348 if (ret) { 1349 NTB_LOG(ERR, "Unable to init ntb dev."); 1350 return ret; 1351 } 1352 1353 if (hw->ntb_ops->set_link == NULL) 1354 return -ENOTSUP; 1355 ret = (*hw->ntb_ops->set_link)(dev, 1); 1356 if (ret) 1357 return ret; 1358 1359 /* Init doorbell. */ 1360 hw->db_valid_mask = RTE_LEN2MASK(hw->db_cnt, uint64_t); 1361 1362 intr_handle = &pci_dev->intr_handle; 1363 /* Register callback func to eal lib */ 1364 rte_intr_callback_register(intr_handle, 1365 ntb_dev_intr_handler, dev); 1366 1367 ret = rte_intr_efd_enable(intr_handle, hw->db_cnt); 1368 if (ret) 1369 return ret; 1370 1371 /* To clarify, the interrupt for each doorbell is already mapped 1372 * by default for intel gen3. They are mapped to msix vec 1-32, 1373 * and hardware intr is mapped to 0. Map all to 0 for uio. 1374 */ 1375 if (!rte_intr_cap_multiple(intr_handle)) { 1376 for (i = 0; i < hw->db_cnt; i++) { 1377 if (hw->ntb_ops->vector_bind == NULL) 1378 return -ENOTSUP; 1379 ret = (*hw->ntb_ops->vector_bind)(dev, i, 0); 1380 if (ret) 1381 return ret; 1382 } 1383 } 1384 1385 if (hw->ntb_ops->db_set_mask == NULL || 1386 hw->ntb_ops->peer_db_set == NULL) { 1387 NTB_LOG(ERR, "Doorbell is not supported."); 1388 return -ENOTSUP; 1389 } 1390 hw->db_mask = 0; 1391 ret = (*hw->ntb_ops->db_set_mask)(dev, hw->db_mask); 1392 if (ret) { 1393 NTB_LOG(ERR, "Unable to enable intr for all dbs."); 1394 return ret; 1395 } 1396 1397 /* enable uio intr after callback register */ 1398 rte_intr_enable(intr_handle); 1399 1400 return ret; 1401 } 1402 1403 static int 1404 ntb_create(struct rte_pci_device *pci_dev, int socket_id) 1405 { 1406 char name[RTE_RAWDEV_NAME_MAX_LEN]; 1407 struct rte_rawdev *rawdev = NULL; 1408 int ret; 1409 1410 if (pci_dev == NULL) { 1411 NTB_LOG(ERR, "Invalid pci_dev."); 1412 return -EINVAL; 1413 } 1414 1415 memset(name, 0, sizeof(name)); 1416 snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "NTB:%x:%02x.%x", 1417 pci_dev->addr.bus, pci_dev->addr.devid, 1418 pci_dev->addr.function); 1419 1420 NTB_LOG(INFO, "Init %s on NUMA node %d", name, socket_id); 1421 1422 /* Allocate device structure. */ 1423 rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct ntb_hw), 1424 socket_id); 1425 if (rawdev == NULL) { 1426 NTB_LOG(ERR, "Unable to allocate rawdev."); 1427 return -EINVAL; 1428 } 1429 1430 rawdev->dev_ops = &ntb_ops; 1431 rawdev->device = &pci_dev->device; 1432 rawdev->driver_name = pci_dev->driver->driver.name; 1433 1434 ret = ntb_init_hw(rawdev, pci_dev); 1435 if (ret < 0) { 1436 NTB_LOG(ERR, "Unable to init ntb hw."); 1437 goto fail; 1438 } 1439 1440 return ret; 1441 1442 fail: 1443 if (rawdev != NULL) 1444 rte_rawdev_pmd_release(rawdev); 1445 1446 return ret; 1447 } 1448 1449 static int 1450 ntb_destroy(struct rte_pci_device *pci_dev) 1451 { 1452 char name[RTE_RAWDEV_NAME_MAX_LEN]; 1453 struct rte_rawdev *rawdev; 1454 int ret; 1455 1456 if (pci_dev == NULL) { 1457 NTB_LOG(ERR, "Invalid pci_dev."); 1458 ret = -EINVAL; 1459 return ret; 1460 } 1461 1462 memset(name, 0, sizeof(name)); 1463 snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "NTB:%x:%02x.%x", 1464 pci_dev->addr.bus, pci_dev->addr.devid, 1465 pci_dev->addr.function); 1466 1467 NTB_LOG(INFO, "Closing %s on NUMA node %d", name, rte_socket_id()); 1468 1469 rawdev = rte_rawdev_pmd_get_named_dev(name); 1470 if (rawdev == NULL) { 1471 NTB_LOG(ERR, "Invalid device name (%s)", name); 1472 ret = -EINVAL; 1473 return ret; 1474 } 1475 1476 ret = rte_rawdev_pmd_release(rawdev); 1477 if (ret) 1478 NTB_LOG(ERR, "Failed to destroy ntb rawdev."); 1479 1480 return ret; 1481 } 1482 1483 static int 1484 ntb_probe(struct rte_pci_driver *pci_drv __rte_unused, 1485 struct rte_pci_device *pci_dev) 1486 { 1487 return ntb_create(pci_dev, rte_socket_id()); 1488 } 1489 1490 static int 1491 ntb_remove(struct rte_pci_device *pci_dev) 1492 { 1493 return ntb_destroy(pci_dev); 1494 } 1495 1496 1497 static struct rte_pci_driver rte_ntb_pmd = { 1498 .id_table = pci_id_ntb_map, 1499 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_WC_ACTIVATE, 1500 .probe = ntb_probe, 1501 .remove = ntb_remove, 1502 }; 1503 1504 RTE_PMD_REGISTER_PCI(raw_ntb, rte_ntb_pmd); 1505 RTE_PMD_REGISTER_PCI_TABLE(raw_ntb, pci_id_ntb_map); 1506 RTE_PMD_REGISTER_KMOD_DEP(raw_ntb, "* igb_uio | uio_pci_generic | vfio-pci"); 1507 1508 RTE_INIT(ntb_init_log) 1509 { 1510 ntb_logtype = rte_log_register("pmd.raw.ntb"); 1511 if (ntb_logtype >= 0) 1512 rte_log_set_level(ntb_logtype, RTE_LOG_INFO); 1513 } 1514