1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2019 Intel Corporation. 3 */ 4 #include <stdint.h> 5 #include <stdio.h> 6 #include <string.h> 7 #include <errno.h> 8 9 #include <rte_common.h> 10 #include <rte_lcore.h> 11 #include <rte_cycles.h> 12 #include <rte_eal.h> 13 #include <rte_log.h> 14 #include <rte_pci.h> 15 #include <rte_mbuf.h> 16 #include <rte_bus_pci.h> 17 #include <rte_memzone.h> 18 #include <rte_memcpy.h> 19 #include <rte_rawdev.h> 20 #include <rte_rawdev_pmd.h> 21 22 #include "ntb_hw_intel.h" 23 #include "rte_pmd_ntb.h" 24 #include "ntb.h" 25 26 static const struct rte_pci_id pci_id_ntb_map[] = { 27 { RTE_PCI_DEVICE(NTB_INTEL_VENDOR_ID, NTB_INTEL_DEV_ID_B2B_SKX) }, 28 { .vendor_id = 0, /* sentinel */ }, 29 }; 30 31 /* Align with enum ntb_xstats_idx */ 32 static struct rte_rawdev_xstats_name ntb_xstats_names[] = { 33 {"Tx-packets"}, 34 {"Tx-bytes"}, 35 {"Tx-errors"}, 36 {"Rx-packets"}, 37 {"Rx-bytes"}, 38 {"Rx-missed"}, 39 }; 40 #define NTB_XSTATS_NUM RTE_DIM(ntb_xstats_names) 41 42 static inline void 43 ntb_link_cleanup(struct rte_rawdev *dev) 44 { 45 struct ntb_hw *hw = dev->dev_private; 46 int status, i; 47 48 if (hw->ntb_ops->spad_write == NULL || 49 hw->ntb_ops->mw_set_trans == NULL) { 50 NTB_LOG(ERR, "Not supported to clean up link."); 51 return; 52 } 53 54 /* Clean spad registers. */ 55 for (i = 0; i < hw->spad_cnt; i++) { 56 status = (*hw->ntb_ops->spad_write)(dev, i, 0, 0); 57 if (status) 58 NTB_LOG(ERR, "Failed to clean local spad."); 59 } 60 61 /* Clear mw so that peer cannot access local memory.*/ 62 for (i = 0; i < hw->used_mw_num; i++) { 63 status = (*hw->ntb_ops->mw_set_trans)(dev, i, 0, 0); 64 if (status) 65 NTB_LOG(ERR, "Failed to clean mw."); 66 } 67 } 68 69 static inline int 70 ntb_handshake_work(const struct rte_rawdev *dev) 71 { 72 struct ntb_hw *hw = dev->dev_private; 73 uint32_t val; 74 int ret, i; 75 76 if (hw->ntb_ops->spad_write == NULL || 77 hw->ntb_ops->mw_set_trans == NULL) { 78 NTB_LOG(ERR, "Scratchpad/MW setting is not supported."); 79 return -ENOTSUP; 80 } 81 82 /* Tell peer the mw info of local side. */ 83 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_NUM_MWS, 1, hw->mw_cnt); 84 if (ret < 0) 85 return ret; 86 for (i = 0; i < hw->mw_cnt; i++) { 87 NTB_LOG(INFO, "Local %u mw size: 0x%"PRIx64"", i, 88 hw->mw_size[i]); 89 val = hw->mw_size[i] >> 32; 90 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_MW0_SZ_H + 2 * i, 91 1, val); 92 if (ret < 0) 93 return ret; 94 val = hw->mw_size[i]; 95 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_MW0_SZ_L + 2 * i, 96 1, val); 97 if (ret < 0) 98 return ret; 99 } 100 101 /* Tell peer about the queue info and map memory to the peer. */ 102 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_Q_SZ, 1, hw->queue_size); 103 if (ret < 0) 104 return ret; 105 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_NUM_QPS, 1, 106 hw->queue_pairs); 107 if (ret < 0) 108 return ret; 109 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_USED_MWS, 1, 110 hw->used_mw_num); 111 if (ret < 0) 112 return ret; 113 for (i = 0; i < hw->used_mw_num; i++) { 114 val = (uint64_t)(size_t)(hw->mz[i]->addr) >> 32; 115 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_MW0_BA_H + 2 * i, 116 1, val); 117 if (ret < 0) 118 return ret; 119 val = (uint64_t)(size_t)(hw->mz[i]->addr); 120 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_MW0_BA_L + 2 * i, 121 1, val); 122 if (ret < 0) 123 return ret; 124 } 125 126 for (i = 0; i < hw->used_mw_num; i++) { 127 ret = (*hw->ntb_ops->mw_set_trans)(dev, i, hw->mz[i]->iova, 128 hw->mz[i]->len); 129 if (ret < 0) 130 return ret; 131 } 132 133 /* Ring doorbell 0 to tell peer the device is ready. */ 134 ret = (*hw->ntb_ops->peer_db_set)(dev, 0); 135 if (ret < 0) 136 return ret; 137 138 return 0; 139 } 140 141 static void 142 ntb_dev_intr_handler(void *param) 143 { 144 struct rte_rawdev *dev = (struct rte_rawdev *)param; 145 struct ntb_hw *hw = dev->dev_private; 146 uint32_t val_h, val_l; 147 uint64_t peer_mw_size; 148 uint64_t db_bits = 0; 149 uint8_t peer_mw_cnt; 150 int i = 0; 151 152 if (hw->ntb_ops->db_read == NULL || 153 hw->ntb_ops->db_clear == NULL || 154 hw->ntb_ops->peer_db_set == NULL) { 155 NTB_LOG(ERR, "Doorbell is not supported."); 156 return; 157 } 158 159 db_bits = (*hw->ntb_ops->db_read)(dev); 160 if (!db_bits) 161 NTB_LOG(ERR, "No doorbells"); 162 163 /* Doorbell 0 is for peer device ready. */ 164 if (db_bits & 1) { 165 NTB_LOG(INFO, "DB0: Peer device is up."); 166 /* Clear received doorbell. */ 167 (*hw->ntb_ops->db_clear)(dev, 1); 168 169 /** 170 * Peer dev is already up. All mw settings are already done. 171 * Skip them. 172 */ 173 if (hw->peer_dev_up) 174 return; 175 176 if (hw->ntb_ops->spad_read == NULL) { 177 NTB_LOG(ERR, "Scratchpad read is not supported."); 178 return; 179 } 180 181 /* Check if mw setting on the peer is the same as local. */ 182 peer_mw_cnt = (*hw->ntb_ops->spad_read)(dev, SPAD_NUM_MWS, 0); 183 if (peer_mw_cnt != hw->mw_cnt) { 184 NTB_LOG(ERR, "Both mw cnt must be the same."); 185 return; 186 } 187 188 for (i = 0; i < hw->mw_cnt; i++) { 189 val_h = (*hw->ntb_ops->spad_read) 190 (dev, SPAD_MW0_SZ_H + 2 * i, 0); 191 val_l = (*hw->ntb_ops->spad_read) 192 (dev, SPAD_MW0_SZ_L + 2 * i, 0); 193 peer_mw_size = ((uint64_t)val_h << 32) | val_l; 194 NTB_LOG(DEBUG, "Peer %u mw size: 0x%"PRIx64"", i, 195 peer_mw_size); 196 if (peer_mw_size != hw->mw_size[i]) { 197 NTB_LOG(ERR, "Mw config must be the same."); 198 return; 199 } 200 } 201 202 hw->peer_dev_up = 1; 203 204 /** 205 * Handshake with peer. Spad_write & mw_set_trans only works 206 * when both devices are up. So write spad again when db is 207 * received. And set db again for the later device who may miss 208 * the 1st db. 209 */ 210 if (ntb_handshake_work(dev) < 0) { 211 NTB_LOG(ERR, "Handshake work failed."); 212 return; 213 } 214 215 /* To get the link info. */ 216 if (hw->ntb_ops->get_link_status == NULL) { 217 NTB_LOG(ERR, "Not supported to get link status."); 218 return; 219 } 220 (*hw->ntb_ops->get_link_status)(dev); 221 NTB_LOG(INFO, "Link is up. Link speed: %u. Link width: %u", 222 hw->link_speed, hw->link_width); 223 return; 224 } 225 226 if (db_bits & (1 << 1)) { 227 NTB_LOG(INFO, "DB1: Peer device is down."); 228 /* Clear received doorbell. */ 229 (*hw->ntb_ops->db_clear)(dev, 2); 230 231 /* Peer device will be down, So clean local side too. */ 232 ntb_link_cleanup(dev); 233 234 hw->peer_dev_up = 0; 235 /* Response peer's dev_stop request. */ 236 (*hw->ntb_ops->peer_db_set)(dev, 2); 237 return; 238 } 239 240 if (db_bits & (1 << 2)) { 241 NTB_LOG(INFO, "DB2: Peer device agrees dev to be down."); 242 /* Clear received doorbell. */ 243 (*hw->ntb_ops->db_clear)(dev, (1 << 2)); 244 hw->peer_dev_up = 0; 245 return; 246 } 247 } 248 249 static int 250 ntb_queue_conf_get(struct rte_rawdev *dev, 251 uint16_t queue_id, 252 rte_rawdev_obj_t queue_conf, 253 size_t conf_size) 254 { 255 struct ntb_queue_conf *q_conf = queue_conf; 256 struct ntb_hw *hw = dev->dev_private; 257 258 if (conf_size != sizeof(*q_conf)) 259 return -EINVAL; 260 261 q_conf->tx_free_thresh = hw->tx_queues[queue_id]->tx_free_thresh; 262 q_conf->nb_desc = hw->rx_queues[queue_id]->nb_rx_desc; 263 q_conf->rx_mp = hw->rx_queues[queue_id]->mpool; 264 265 return 0; 266 } 267 268 static void 269 ntb_rxq_release_mbufs(struct ntb_rx_queue *q) 270 { 271 int i; 272 273 if (!q || !q->sw_ring) { 274 NTB_LOG(ERR, "Pointer to rxq or sw_ring is NULL"); 275 return; 276 } 277 278 for (i = 0; i < q->nb_rx_desc; i++) { 279 if (q->sw_ring[i].mbuf) { 280 rte_pktmbuf_free_seg(q->sw_ring[i].mbuf); 281 q->sw_ring[i].mbuf = NULL; 282 } 283 } 284 } 285 286 static void 287 ntb_rxq_release(struct ntb_rx_queue *rxq) 288 { 289 if (!rxq) { 290 NTB_LOG(ERR, "Pointer to rxq is NULL"); 291 return; 292 } 293 294 ntb_rxq_release_mbufs(rxq); 295 296 rte_free(rxq->sw_ring); 297 rte_free(rxq); 298 } 299 300 static int 301 ntb_rxq_setup(struct rte_rawdev *dev, 302 uint16_t qp_id, 303 rte_rawdev_obj_t queue_conf, 304 size_t conf_size) 305 { 306 struct ntb_queue_conf *rxq_conf = queue_conf; 307 struct ntb_hw *hw = dev->dev_private; 308 struct ntb_rx_queue *rxq; 309 310 if (conf_size != sizeof(*rxq_conf)) 311 return -EINVAL; 312 313 /* Allocate the rx queue data structure */ 314 rxq = rte_zmalloc_socket("ntb rx queue", 315 sizeof(struct ntb_rx_queue), 316 RTE_CACHE_LINE_SIZE, 317 dev->socket_id); 318 if (!rxq) { 319 NTB_LOG(ERR, "Failed to allocate memory for " 320 "rx queue data structure."); 321 return -ENOMEM; 322 } 323 324 if (rxq_conf->rx_mp == NULL) { 325 NTB_LOG(ERR, "Invalid null mempool pointer."); 326 return -EINVAL; 327 } 328 rxq->nb_rx_desc = rxq_conf->nb_desc; 329 rxq->mpool = rxq_conf->rx_mp; 330 rxq->port_id = dev->dev_id; 331 rxq->queue_id = qp_id; 332 rxq->hw = hw; 333 334 /* Allocate the software ring. */ 335 rxq->sw_ring = 336 rte_zmalloc_socket("ntb rx sw ring", 337 sizeof(struct ntb_rx_entry) * 338 rxq->nb_rx_desc, 339 RTE_CACHE_LINE_SIZE, 340 dev->socket_id); 341 if (!rxq->sw_ring) { 342 ntb_rxq_release(rxq); 343 rxq = NULL; 344 NTB_LOG(ERR, "Failed to allocate memory for SW ring"); 345 return -ENOMEM; 346 } 347 348 hw->rx_queues[qp_id] = rxq; 349 350 return 0; 351 } 352 353 static void 354 ntb_txq_release_mbufs(struct ntb_tx_queue *q) 355 { 356 int i; 357 358 if (!q || !q->sw_ring) { 359 NTB_LOG(ERR, "Pointer to txq or sw_ring is NULL"); 360 return; 361 } 362 363 for (i = 0; i < q->nb_tx_desc; i++) { 364 if (q->sw_ring[i].mbuf) { 365 rte_pktmbuf_free_seg(q->sw_ring[i].mbuf); 366 q->sw_ring[i].mbuf = NULL; 367 } 368 } 369 } 370 371 static void 372 ntb_txq_release(struct ntb_tx_queue *txq) 373 { 374 if (!txq) { 375 NTB_LOG(ERR, "Pointer to txq is NULL"); 376 return; 377 } 378 379 ntb_txq_release_mbufs(txq); 380 381 rte_free(txq->sw_ring); 382 rte_free(txq); 383 } 384 385 static int 386 ntb_txq_setup(struct rte_rawdev *dev, 387 uint16_t qp_id, 388 rte_rawdev_obj_t queue_conf, 389 size_t conf_size) 390 { 391 struct ntb_queue_conf *txq_conf = queue_conf; 392 struct ntb_hw *hw = dev->dev_private; 393 struct ntb_tx_queue *txq; 394 uint16_t i, prev; 395 396 if (conf_size != sizeof(*txq_conf)) 397 return -EINVAL; 398 399 /* Allocate the TX queue data structure. */ 400 txq = rte_zmalloc_socket("ntb tx queue", 401 sizeof(struct ntb_tx_queue), 402 RTE_CACHE_LINE_SIZE, 403 dev->socket_id); 404 if (!txq) { 405 NTB_LOG(ERR, "Failed to allocate memory for " 406 "tx queue structure"); 407 return -ENOMEM; 408 } 409 410 txq->nb_tx_desc = txq_conf->nb_desc; 411 txq->port_id = dev->dev_id; 412 txq->queue_id = qp_id; 413 txq->hw = hw; 414 415 /* Allocate software ring */ 416 txq->sw_ring = 417 rte_zmalloc_socket("ntb tx sw ring", 418 sizeof(struct ntb_tx_entry) * 419 txq->nb_tx_desc, 420 RTE_CACHE_LINE_SIZE, 421 dev->socket_id); 422 if (!txq->sw_ring) { 423 ntb_txq_release(txq); 424 txq = NULL; 425 NTB_LOG(ERR, "Failed to allocate memory for SW TX ring"); 426 return -ENOMEM; 427 } 428 429 prev = txq->nb_tx_desc - 1; 430 for (i = 0; i < txq->nb_tx_desc; i++) { 431 txq->sw_ring[i].mbuf = NULL; 432 txq->sw_ring[i].last_id = i; 433 txq->sw_ring[prev].next_id = i; 434 prev = i; 435 } 436 437 txq->tx_free_thresh = txq_conf->tx_free_thresh ? 438 txq_conf->tx_free_thresh : 439 NTB_DFLT_TX_FREE_THRESH; 440 if (txq->tx_free_thresh >= txq->nb_tx_desc - 3) { 441 NTB_LOG(ERR, "tx_free_thresh must be less than nb_desc - 3. " 442 "(tx_free_thresh=%u qp_id=%u)", txq->tx_free_thresh, 443 qp_id); 444 return -EINVAL; 445 } 446 447 hw->tx_queues[qp_id] = txq; 448 449 return 0; 450 } 451 452 453 static int 454 ntb_queue_setup(struct rte_rawdev *dev, 455 uint16_t queue_id, 456 rte_rawdev_obj_t queue_conf, 457 size_t conf_size) 458 { 459 struct ntb_hw *hw = dev->dev_private; 460 int ret; 461 462 if (queue_id >= hw->queue_pairs) 463 return -EINVAL; 464 465 ret = ntb_txq_setup(dev, queue_id, queue_conf, conf_size); 466 if (ret < 0) 467 return ret; 468 469 ret = ntb_rxq_setup(dev, queue_id, queue_conf, conf_size); 470 471 return ret; 472 } 473 474 static int 475 ntb_queue_release(struct rte_rawdev *dev, uint16_t queue_id) 476 { 477 struct ntb_hw *hw = dev->dev_private; 478 479 if (queue_id >= hw->queue_pairs) 480 return -EINVAL; 481 482 ntb_txq_release(hw->tx_queues[queue_id]); 483 hw->tx_queues[queue_id] = NULL; 484 ntb_rxq_release(hw->rx_queues[queue_id]); 485 hw->rx_queues[queue_id] = NULL; 486 487 return 0; 488 } 489 490 static uint16_t 491 ntb_queue_count(struct rte_rawdev *dev) 492 { 493 struct ntb_hw *hw = dev->dev_private; 494 return hw->queue_pairs; 495 } 496 497 static int 498 ntb_queue_init(struct rte_rawdev *dev, uint16_t qp_id) 499 { 500 struct ntb_hw *hw = dev->dev_private; 501 struct ntb_rx_queue *rxq = hw->rx_queues[qp_id]; 502 struct ntb_tx_queue *txq = hw->tx_queues[qp_id]; 503 volatile struct ntb_header *local_hdr; 504 struct ntb_header *remote_hdr; 505 uint16_t q_size = hw->queue_size; 506 uint32_t hdr_offset; 507 void *bar_addr; 508 uint16_t i; 509 510 if (hw->ntb_ops->get_peer_mw_addr == NULL) { 511 NTB_LOG(ERR, "Getting peer mw addr is not supported."); 512 return -EINVAL; 513 } 514 515 /* Put queue info into the start of shared memory. */ 516 hdr_offset = hw->hdr_size_per_queue * qp_id; 517 local_hdr = (volatile struct ntb_header *) 518 ((size_t)hw->mz[0]->addr + hdr_offset); 519 bar_addr = (*hw->ntb_ops->get_peer_mw_addr)(dev, 0); 520 if (bar_addr == NULL) 521 return -EINVAL; 522 remote_hdr = (struct ntb_header *) 523 ((size_t)bar_addr + hdr_offset); 524 525 /* rxq init. */ 526 rxq->rx_desc_ring = (struct ntb_desc *) 527 (&remote_hdr->desc_ring); 528 rxq->rx_used_ring = (volatile struct ntb_used *) 529 (&local_hdr->desc_ring[q_size]); 530 rxq->avail_cnt = &remote_hdr->avail_cnt; 531 rxq->used_cnt = &local_hdr->used_cnt; 532 533 for (i = 0; i < rxq->nb_rx_desc - 1; i++) { 534 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mpool); 535 if (unlikely(!mbuf)) { 536 NTB_LOG(ERR, "Failed to allocate mbuf for RX"); 537 return -ENOMEM; 538 } 539 mbuf->port = dev->dev_id; 540 541 rxq->sw_ring[i].mbuf = mbuf; 542 543 rxq->rx_desc_ring[i].addr = rte_pktmbuf_mtod(mbuf, size_t); 544 rxq->rx_desc_ring[i].len = mbuf->buf_len - RTE_PKTMBUF_HEADROOM; 545 } 546 rte_wmb(); 547 *rxq->avail_cnt = rxq->nb_rx_desc - 1; 548 rxq->last_avail = rxq->nb_rx_desc - 1; 549 rxq->last_used = 0; 550 551 /* txq init */ 552 txq->tx_desc_ring = (volatile struct ntb_desc *) 553 (&local_hdr->desc_ring); 554 txq->tx_used_ring = (struct ntb_used *) 555 (&remote_hdr->desc_ring[q_size]); 556 txq->avail_cnt = &local_hdr->avail_cnt; 557 txq->used_cnt = &remote_hdr->used_cnt; 558 559 rte_wmb(); 560 *txq->used_cnt = 0; 561 txq->last_used = 0; 562 txq->last_avail = 0; 563 txq->nb_tx_free = txq->nb_tx_desc - 1; 564 565 /* Set per queue stats. */ 566 for (i = 0; i < NTB_XSTATS_NUM; i++) { 567 hw->ntb_xstats[i + NTB_XSTATS_NUM * (qp_id + 1)] = 0; 568 hw->ntb_xstats_off[i + NTB_XSTATS_NUM * (qp_id + 1)] = 0; 569 } 570 571 return 0; 572 } 573 574 static inline void 575 ntb_enqueue_cleanup(struct ntb_tx_queue *txq) 576 { 577 struct ntb_tx_entry *sw_ring = txq->sw_ring; 578 uint16_t tx_free = txq->last_avail; 579 uint16_t nb_to_clean, i; 580 581 /* avail_cnt + 1 represents where to rx next in the peer. */ 582 nb_to_clean = (*txq->avail_cnt - txq->last_avail + 1 + 583 txq->nb_tx_desc) & (txq->nb_tx_desc - 1); 584 nb_to_clean = RTE_MIN(nb_to_clean, txq->tx_free_thresh); 585 for (i = 0; i < nb_to_clean; i++) { 586 if (sw_ring[tx_free].mbuf) 587 rte_pktmbuf_free_seg(sw_ring[tx_free].mbuf); 588 tx_free = (tx_free + 1) & (txq->nb_tx_desc - 1); 589 } 590 591 txq->nb_tx_free += nb_to_clean; 592 txq->last_avail = tx_free; 593 } 594 595 static int 596 ntb_enqueue_bufs(struct rte_rawdev *dev, 597 struct rte_rawdev_buf **buffers, 598 unsigned int count, 599 rte_rawdev_obj_t context) 600 { 601 struct ntb_hw *hw = dev->dev_private; 602 struct ntb_tx_queue *txq = hw->tx_queues[(size_t)context]; 603 struct ntb_tx_entry *sw_ring = txq->sw_ring; 604 struct rte_mbuf *txm; 605 struct ntb_used tx_used[NTB_MAX_DESC_SIZE]; 606 volatile struct ntb_desc *tx_item; 607 uint16_t tx_last, nb_segs, off, last_used, avail_cnt; 608 uint16_t nb_mbufs = 0; 609 uint16_t nb_tx = 0; 610 uint64_t bytes = 0; 611 void *buf_addr; 612 int i; 613 614 if (unlikely(hw->ntb_ops->ioremap == NULL)) { 615 NTB_LOG(ERR, "Ioremap not supported."); 616 return nb_tx; 617 } 618 619 if (unlikely(dev->started == 0 || hw->peer_dev_up == 0)) { 620 NTB_LOG(DEBUG, "Link is not up."); 621 return nb_tx; 622 } 623 624 if (txq->nb_tx_free < txq->tx_free_thresh) 625 ntb_enqueue_cleanup(txq); 626 627 off = NTB_XSTATS_NUM * ((size_t)context + 1); 628 last_used = txq->last_used; 629 avail_cnt = *txq->avail_cnt;/* Where to alloc next. */ 630 for (nb_tx = 0; nb_tx < count; nb_tx++) { 631 txm = (struct rte_mbuf *)(buffers[nb_tx]->buf_addr); 632 if (txm == NULL || txq->nb_tx_free < txm->nb_segs) 633 break; 634 635 tx_last = (txq->last_used + txm->nb_segs - 1) & 636 (txq->nb_tx_desc - 1); 637 nb_segs = txm->nb_segs; 638 for (i = 0; i < nb_segs; i++) { 639 /* Not enough ring space for tx. */ 640 if (txq->last_used == avail_cnt) 641 goto end_of_tx; 642 sw_ring[txq->last_used].mbuf = txm; 643 tx_item = txq->tx_desc_ring + txq->last_used; 644 645 if (!tx_item->len) { 646 (hw->ntb_xstats[NTB_TX_ERRS_ID + off])++; 647 goto end_of_tx; 648 } 649 if (txm->data_len > tx_item->len) { 650 NTB_LOG(ERR, "Data length exceeds buf length." 651 " Only %u data would be transmitted.", 652 tx_item->len); 653 txm->data_len = tx_item->len; 654 } 655 656 /* translate remote virtual addr to bar virtual addr */ 657 buf_addr = (*hw->ntb_ops->ioremap)(dev, tx_item->addr); 658 if (buf_addr == NULL) { 659 (hw->ntb_xstats[NTB_TX_ERRS_ID + off])++; 660 NTB_LOG(ERR, "Null remap addr."); 661 goto end_of_tx; 662 } 663 rte_memcpy(buf_addr, rte_pktmbuf_mtod(txm, void *), 664 txm->data_len); 665 666 tx_used[nb_mbufs].len = txm->data_len; 667 tx_used[nb_mbufs++].flags = (txq->last_used == 668 tx_last) ? 669 NTB_FLAG_EOP : 0; 670 671 /* update stats */ 672 bytes += txm->data_len; 673 674 txm = txm->next; 675 676 sw_ring[txq->last_used].next_id = (txq->last_used + 1) & 677 (txq->nb_tx_desc - 1); 678 sw_ring[txq->last_used].last_id = tx_last; 679 txq->last_used = (txq->last_used + 1) & 680 (txq->nb_tx_desc - 1); 681 } 682 txq->nb_tx_free -= nb_segs; 683 } 684 685 end_of_tx: 686 if (nb_tx) { 687 uint16_t nb1, nb2; 688 if (nb_mbufs > txq->nb_tx_desc - last_used) { 689 nb1 = txq->nb_tx_desc - last_used; 690 nb2 = nb_mbufs - txq->nb_tx_desc + last_used; 691 } else { 692 nb1 = nb_mbufs; 693 nb2 = 0; 694 } 695 rte_memcpy(txq->tx_used_ring + last_used, tx_used, 696 sizeof(struct ntb_used) * nb1); 697 rte_memcpy(txq->tx_used_ring, tx_used + nb1, 698 sizeof(struct ntb_used) * nb2); 699 rte_wmb(); 700 *txq->used_cnt = txq->last_used; 701 702 /* update queue stats */ 703 hw->ntb_xstats[NTB_TX_BYTES_ID + off] += bytes; 704 hw->ntb_xstats[NTB_TX_PKTS_ID + off] += nb_tx; 705 } 706 707 return nb_tx; 708 } 709 710 static int 711 ntb_dequeue_bufs(struct rte_rawdev *dev, 712 struct rte_rawdev_buf **buffers, 713 unsigned int count, 714 rte_rawdev_obj_t context) 715 { 716 struct ntb_hw *hw = dev->dev_private; 717 struct ntb_rx_queue *rxq = hw->rx_queues[(size_t)context]; 718 struct ntb_rx_entry *sw_ring = rxq->sw_ring; 719 struct ntb_desc rx_desc[NTB_MAX_DESC_SIZE]; 720 struct rte_mbuf *first, *rxm_t; 721 struct rte_mbuf *prev = NULL; 722 volatile struct ntb_used *rx_item; 723 uint16_t nb_mbufs = 0; 724 uint16_t nb_rx = 0; 725 uint64_t bytes = 0; 726 uint16_t off, last_avail, used_cnt, used_nb; 727 int i; 728 729 if (unlikely(dev->started == 0 || hw->peer_dev_up == 0)) { 730 NTB_LOG(DEBUG, "Link is not up"); 731 return nb_rx; 732 } 733 734 used_cnt = *rxq->used_cnt; 735 736 if (rxq->last_used == used_cnt) 737 return nb_rx; 738 739 last_avail = rxq->last_avail; 740 used_nb = (used_cnt - rxq->last_used) & (rxq->nb_rx_desc - 1); 741 count = RTE_MIN(count, used_nb); 742 for (nb_rx = 0; nb_rx < count; nb_rx++) { 743 i = 0; 744 while (true) { 745 rx_item = rxq->rx_used_ring + rxq->last_used; 746 rxm_t = sw_ring[rxq->last_used].mbuf; 747 rxm_t->data_len = rx_item->len; 748 rxm_t->data_off = RTE_PKTMBUF_HEADROOM; 749 rxm_t->port = rxq->port_id; 750 751 if (!i) { 752 rxm_t->nb_segs = 1; 753 first = rxm_t; 754 first->pkt_len = 0; 755 buffers[nb_rx]->buf_addr = rxm_t; 756 } else { 757 prev->next = rxm_t; 758 first->nb_segs++; 759 } 760 761 prev = rxm_t; 762 first->pkt_len += prev->data_len; 763 rxq->last_used = (rxq->last_used + 1) & 764 (rxq->nb_rx_desc - 1); 765 766 /* alloc new mbuf */ 767 rxm_t = rte_mbuf_raw_alloc(rxq->mpool); 768 if (unlikely(rxm_t == NULL)) { 769 NTB_LOG(ERR, "recv alloc mbuf failed."); 770 goto end_of_rx; 771 } 772 rxm_t->port = rxq->port_id; 773 sw_ring[rxq->last_avail].mbuf = rxm_t; 774 i++; 775 776 /* fill new desc */ 777 rx_desc[nb_mbufs].addr = 778 rte_pktmbuf_mtod(rxm_t, size_t); 779 rx_desc[nb_mbufs++].len = rxm_t->buf_len - 780 RTE_PKTMBUF_HEADROOM; 781 rxq->last_avail = (rxq->last_avail + 1) & 782 (rxq->nb_rx_desc - 1); 783 784 if (rx_item->flags & NTB_FLAG_EOP) 785 break; 786 } 787 /* update stats */ 788 bytes += first->pkt_len; 789 } 790 791 end_of_rx: 792 if (nb_rx) { 793 uint16_t nb1, nb2; 794 if (nb_mbufs > rxq->nb_rx_desc - last_avail) { 795 nb1 = rxq->nb_rx_desc - last_avail; 796 nb2 = nb_mbufs - rxq->nb_rx_desc + last_avail; 797 } else { 798 nb1 = nb_mbufs; 799 nb2 = 0; 800 } 801 rte_memcpy(rxq->rx_desc_ring + last_avail, rx_desc, 802 sizeof(struct ntb_desc) * nb1); 803 rte_memcpy(rxq->rx_desc_ring, rx_desc + nb1, 804 sizeof(struct ntb_desc) * nb2); 805 rte_wmb(); 806 *rxq->avail_cnt = rxq->last_avail; 807 808 /* update queue stats */ 809 off = NTB_XSTATS_NUM * ((size_t)context + 1); 810 hw->ntb_xstats[NTB_RX_BYTES_ID + off] += bytes; 811 hw->ntb_xstats[NTB_RX_PKTS_ID + off] += nb_rx; 812 hw->ntb_xstats[NTB_RX_MISS_ID + off] += (count - nb_rx); 813 } 814 815 return nb_rx; 816 } 817 818 static int 819 ntb_dev_info_get(struct rte_rawdev *dev, rte_rawdev_obj_t dev_info, 820 size_t dev_info_size) 821 { 822 struct ntb_hw *hw = dev->dev_private; 823 struct ntb_dev_info *info = dev_info; 824 825 if (dev_info_size != sizeof(*info)) { 826 NTB_LOG(ERR, "Invalid size parameter to %s", __func__); 827 return -EINVAL; 828 } 829 830 info->mw_cnt = hw->mw_cnt; 831 info->mw_size = hw->mw_size; 832 833 /** 834 * Intel hardware requires that mapped memory base address should be 835 * aligned with EMBARSZ and needs continuous memzone. 836 */ 837 info->mw_size_align = (uint8_t)(hw->pci_dev->id.vendor_id == 838 NTB_INTEL_VENDOR_ID); 839 840 if (!hw->queue_size || !hw->queue_pairs) { 841 NTB_LOG(ERR, "No queue size and queue num assigned."); 842 return -EAGAIN; 843 } 844 845 hw->hdr_size_per_queue = RTE_ALIGN(sizeof(struct ntb_header) + 846 hw->queue_size * sizeof(struct ntb_desc) + 847 hw->queue_size * sizeof(struct ntb_used), 848 RTE_CACHE_LINE_SIZE); 849 info->ntb_hdr_size = hw->hdr_size_per_queue * hw->queue_pairs; 850 851 return 0; 852 } 853 854 static int 855 ntb_dev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config, 856 size_t config_size) 857 { 858 struct ntb_dev_config *conf = config; 859 struct ntb_hw *hw = dev->dev_private; 860 uint32_t xstats_num; 861 int ret; 862 863 if (conf == NULL || config_size != sizeof(*conf)) 864 return -EINVAL; 865 866 hw->queue_pairs = conf->num_queues; 867 hw->queue_size = conf->queue_size; 868 hw->used_mw_num = conf->mz_num; 869 hw->mz = conf->mz_list; 870 hw->rx_queues = rte_zmalloc("ntb_rx_queues", 871 sizeof(struct ntb_rx_queue *) * hw->queue_pairs, 0); 872 hw->tx_queues = rte_zmalloc("ntb_tx_queues", 873 sizeof(struct ntb_tx_queue *) * hw->queue_pairs, 0); 874 /* First total stats, then per queue stats. */ 875 xstats_num = (hw->queue_pairs + 1) * NTB_XSTATS_NUM; 876 hw->ntb_xstats = rte_zmalloc("ntb_xstats", xstats_num * 877 sizeof(uint64_t), 0); 878 hw->ntb_xstats_off = rte_zmalloc("ntb_xstats_off", xstats_num * 879 sizeof(uint64_t), 0); 880 881 /* Start handshake with the peer. */ 882 ret = ntb_handshake_work(dev); 883 if (ret < 0) { 884 rte_free(hw->rx_queues); 885 rte_free(hw->tx_queues); 886 hw->rx_queues = NULL; 887 hw->tx_queues = NULL; 888 return ret; 889 } 890 891 return 0; 892 } 893 894 static int 895 ntb_dev_start(struct rte_rawdev *dev) 896 { 897 struct ntb_hw *hw = dev->dev_private; 898 uint32_t peer_base_l, peer_val; 899 uint64_t peer_base_h; 900 uint32_t i; 901 int ret; 902 903 if (!hw->link_status || !hw->peer_dev_up) 904 return -EINVAL; 905 906 /* Set total stats. */ 907 for (i = 0; i < NTB_XSTATS_NUM; i++) { 908 hw->ntb_xstats[i] = 0; 909 hw->ntb_xstats_off[i] = 0; 910 } 911 912 for (i = 0; i < hw->queue_pairs; i++) { 913 ret = ntb_queue_init(dev, i); 914 if (ret) { 915 NTB_LOG(ERR, "Failed to init queue."); 916 goto err_q_init; 917 } 918 } 919 920 hw->peer_mw_base = rte_zmalloc("ntb_peer_mw_base", hw->mw_cnt * 921 sizeof(uint64_t), 0); 922 923 if (hw->ntb_ops->spad_read == NULL) { 924 ret = -ENOTSUP; 925 goto err_up; 926 } 927 928 peer_val = (*hw->ntb_ops->spad_read)(dev, SPAD_Q_SZ, 0); 929 if (peer_val != hw->queue_size) { 930 NTB_LOG(ERR, "Inconsistent queue size! (local: %u peer: %u)", 931 hw->queue_size, peer_val); 932 ret = -EINVAL; 933 goto err_up; 934 } 935 936 peer_val = (*hw->ntb_ops->spad_read)(dev, SPAD_NUM_QPS, 0); 937 if (peer_val != hw->queue_pairs) { 938 NTB_LOG(ERR, "Inconsistent number of queues! (local: %u peer:" 939 " %u)", hw->queue_pairs, peer_val); 940 ret = -EINVAL; 941 goto err_up; 942 } 943 944 hw->peer_used_mws = (*hw->ntb_ops->spad_read)(dev, SPAD_USED_MWS, 0); 945 946 for (i = 0; i < hw->peer_used_mws; i++) { 947 peer_base_h = (*hw->ntb_ops->spad_read)(dev, 948 SPAD_MW0_BA_H + 2 * i, 0); 949 peer_base_l = (*hw->ntb_ops->spad_read)(dev, 950 SPAD_MW0_BA_L + 2 * i, 0); 951 hw->peer_mw_base[i] = (peer_base_h << 32) + peer_base_l; 952 } 953 954 dev->started = 1; 955 956 return 0; 957 958 err_up: 959 rte_free(hw->peer_mw_base); 960 err_q_init: 961 for (i = 0; i < hw->queue_pairs; i++) { 962 ntb_rxq_release_mbufs(hw->rx_queues[i]); 963 ntb_txq_release_mbufs(hw->tx_queues[i]); 964 } 965 966 return ret; 967 } 968 969 static void 970 ntb_dev_stop(struct rte_rawdev *dev) 971 { 972 struct ntb_hw *hw = dev->dev_private; 973 uint32_t time_out; 974 int status, i; 975 976 if (!hw->peer_dev_up) 977 goto clean; 978 979 ntb_link_cleanup(dev); 980 981 /* Notify the peer that device will be down. */ 982 if (hw->ntb_ops->peer_db_set == NULL) { 983 NTB_LOG(ERR, "Peer doorbell setting is not supported."); 984 return; 985 } 986 status = (*hw->ntb_ops->peer_db_set)(dev, 1); 987 if (status) { 988 NTB_LOG(ERR, "Failed to tell peer device is down."); 989 return; 990 } 991 992 /* 993 * Set time out as 1s in case that the peer is stopped accidently 994 * without any notification. 995 */ 996 time_out = 1000000; 997 998 /* Wait for cleanup work down before db mask clear. */ 999 while (hw->peer_dev_up && time_out) { 1000 time_out -= 10; 1001 rte_delay_us(10); 1002 } 1003 1004 clean: 1005 /* Clear doorbells mask. */ 1006 if (hw->ntb_ops->db_set_mask == NULL) { 1007 NTB_LOG(ERR, "Doorbell mask setting is not supported."); 1008 return; 1009 } 1010 status = (*hw->ntb_ops->db_set_mask)(dev, 1011 (((uint64_t)1 << hw->db_cnt) - 1)); 1012 if (status) 1013 NTB_LOG(ERR, "Failed to clear doorbells."); 1014 1015 for (i = 0; i < hw->queue_pairs; i++) { 1016 ntb_rxq_release_mbufs(hw->rx_queues[i]); 1017 ntb_txq_release_mbufs(hw->tx_queues[i]); 1018 } 1019 1020 dev->started = 0; 1021 } 1022 1023 static int 1024 ntb_dev_close(struct rte_rawdev *dev) 1025 { 1026 struct ntb_hw *hw = dev->dev_private; 1027 struct rte_intr_handle *intr_handle; 1028 int i; 1029 1030 if (dev->started) 1031 ntb_dev_stop(dev); 1032 1033 /* free queues */ 1034 for (i = 0; i < hw->queue_pairs; i++) 1035 ntb_queue_release(dev, i); 1036 hw->queue_pairs = 0; 1037 1038 intr_handle = &hw->pci_dev->intr_handle; 1039 /* Clean datapath event and vec mapping */ 1040 rte_intr_efd_disable(intr_handle); 1041 if (intr_handle->intr_vec) { 1042 rte_free(intr_handle->intr_vec); 1043 intr_handle->intr_vec = NULL; 1044 } 1045 /* Disable uio intr before callback unregister */ 1046 rte_intr_disable(intr_handle); 1047 1048 /* Unregister callback func to eal lib */ 1049 rte_intr_callback_unregister(intr_handle, 1050 ntb_dev_intr_handler, dev); 1051 1052 return 0; 1053 } 1054 1055 static int 1056 ntb_dev_reset(struct rte_rawdev *rawdev __rte_unused) 1057 { 1058 return 0; 1059 } 1060 1061 static int 1062 ntb_attr_set(struct rte_rawdev *dev, const char *attr_name, 1063 uint64_t attr_value) 1064 { 1065 struct ntb_hw *hw; 1066 int index; 1067 1068 if (dev == NULL || attr_name == NULL) { 1069 NTB_LOG(ERR, "Invalid arguments for setting attributes"); 1070 return -EINVAL; 1071 } 1072 1073 hw = dev->dev_private; 1074 1075 if (!strncmp(attr_name, NTB_SPAD_USER, NTB_SPAD_USER_LEN)) { 1076 if (hw->ntb_ops->spad_write == NULL) 1077 return -ENOTSUP; 1078 index = atoi(&attr_name[NTB_SPAD_USER_LEN]); 1079 (*hw->ntb_ops->spad_write)(dev, hw->spad_user_list[index], 1080 1, attr_value); 1081 NTB_LOG(DEBUG, "Set attribute (%s) Value (%" PRIu64 ")", 1082 attr_name, attr_value); 1083 return 0; 1084 } 1085 1086 if (!strncmp(attr_name, NTB_QUEUE_SZ_NAME, NTB_ATTR_NAME_LEN)) { 1087 hw->queue_size = attr_value; 1088 NTB_LOG(DEBUG, "Set attribute (%s) Value (%" PRIu64 ")", 1089 attr_name, attr_value); 1090 return 0; 1091 } 1092 1093 if (!strncmp(attr_name, NTB_QUEUE_NUM_NAME, NTB_ATTR_NAME_LEN)) { 1094 hw->queue_pairs = attr_value; 1095 NTB_LOG(DEBUG, "Set attribute (%s) Value (%" PRIu64 ")", 1096 attr_name, attr_value); 1097 return 0; 1098 } 1099 1100 /* Attribute not found. */ 1101 NTB_LOG(ERR, "Attribute not found."); 1102 return -EINVAL; 1103 } 1104 1105 static int 1106 ntb_attr_get(struct rte_rawdev *dev, const char *attr_name, 1107 uint64_t *attr_value) 1108 { 1109 struct ntb_hw *hw; 1110 int index; 1111 1112 if (dev == NULL || attr_name == NULL || attr_value == NULL) { 1113 NTB_LOG(ERR, "Invalid arguments for getting attributes"); 1114 return -EINVAL; 1115 } 1116 1117 hw = dev->dev_private; 1118 1119 if (!strncmp(attr_name, NTB_TOPO_NAME, NTB_ATTR_NAME_LEN)) { 1120 *attr_value = hw->topo; 1121 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")", 1122 attr_name, *attr_value); 1123 return 0; 1124 } 1125 1126 if (!strncmp(attr_name, NTB_LINK_STATUS_NAME, NTB_ATTR_NAME_LEN)) { 1127 /* hw->link_status only indicates hw link status. */ 1128 *attr_value = hw->link_status && hw->peer_dev_up; 1129 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")", 1130 attr_name, *attr_value); 1131 return 0; 1132 } 1133 1134 if (!strncmp(attr_name, NTB_SPEED_NAME, NTB_ATTR_NAME_LEN)) { 1135 *attr_value = hw->link_speed; 1136 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")", 1137 attr_name, *attr_value); 1138 return 0; 1139 } 1140 1141 if (!strncmp(attr_name, NTB_WIDTH_NAME, NTB_ATTR_NAME_LEN)) { 1142 *attr_value = hw->link_width; 1143 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")", 1144 attr_name, *attr_value); 1145 return 0; 1146 } 1147 1148 if (!strncmp(attr_name, NTB_MW_CNT_NAME, NTB_ATTR_NAME_LEN)) { 1149 *attr_value = hw->mw_cnt; 1150 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")", 1151 attr_name, *attr_value); 1152 return 0; 1153 } 1154 1155 if (!strncmp(attr_name, NTB_DB_CNT_NAME, NTB_ATTR_NAME_LEN)) { 1156 *attr_value = hw->db_cnt; 1157 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")", 1158 attr_name, *attr_value); 1159 return 0; 1160 } 1161 1162 if (!strncmp(attr_name, NTB_SPAD_CNT_NAME, NTB_ATTR_NAME_LEN)) { 1163 *attr_value = hw->spad_cnt; 1164 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")", 1165 attr_name, *attr_value); 1166 return 0; 1167 } 1168 1169 if (!strncmp(attr_name, NTB_SPAD_USER, NTB_SPAD_USER_LEN)) { 1170 if (hw->ntb_ops->spad_read == NULL) 1171 return -ENOTSUP; 1172 index = atoi(&attr_name[NTB_SPAD_USER_LEN]); 1173 *attr_value = (*hw->ntb_ops->spad_read)(dev, 1174 hw->spad_user_list[index], 0); 1175 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")", 1176 attr_name, *attr_value); 1177 return 0; 1178 } 1179 1180 /* Attribute not found. */ 1181 NTB_LOG(ERR, "Attribute not found."); 1182 return -EINVAL; 1183 } 1184 1185 static inline uint64_t 1186 ntb_stats_update(uint64_t offset, uint64_t stat) 1187 { 1188 if (stat >= offset) 1189 return (stat - offset); 1190 else 1191 return (uint64_t)(((uint64_t)-1) - offset + stat + 1); 1192 } 1193 1194 static int 1195 ntb_xstats_get(const struct rte_rawdev *dev, 1196 const unsigned int ids[], 1197 uint64_t values[], 1198 unsigned int n) 1199 { 1200 struct ntb_hw *hw = dev->dev_private; 1201 uint32_t i, j, off, xstats_num; 1202 1203 /* Calculate total stats of all queues. */ 1204 for (i = 0; i < NTB_XSTATS_NUM; i++) { 1205 hw->ntb_xstats[i] = 0; 1206 for (j = 0; j < hw->queue_pairs; j++) { 1207 off = NTB_XSTATS_NUM * (j + 1) + i; 1208 hw->ntb_xstats[i] += 1209 ntb_stats_update(hw->ntb_xstats_off[off], 1210 hw->ntb_xstats[off]); 1211 } 1212 } 1213 1214 xstats_num = NTB_XSTATS_NUM * (hw->queue_pairs + 1); 1215 for (i = 0; i < n && ids[i] < xstats_num; i++) { 1216 if (ids[i] < NTB_XSTATS_NUM) 1217 values[i] = hw->ntb_xstats[ids[i]]; 1218 else 1219 values[i] = 1220 ntb_stats_update(hw->ntb_xstats_off[ids[i]], 1221 hw->ntb_xstats[ids[i]]); 1222 } 1223 1224 return i; 1225 } 1226 1227 static int 1228 ntb_xstats_get_names(const struct rte_rawdev *dev, 1229 struct rte_rawdev_xstats_name *xstats_names, 1230 unsigned int size) 1231 { 1232 struct ntb_hw *hw = dev->dev_private; 1233 uint32_t xstats_num, i, j, off; 1234 1235 xstats_num = NTB_XSTATS_NUM * (hw->queue_pairs + 1); 1236 if (xstats_names == NULL || size < xstats_num) 1237 return xstats_num; 1238 1239 /* Total stats names */ 1240 memcpy(xstats_names, ntb_xstats_names, sizeof(ntb_xstats_names)); 1241 1242 /* Queue stats names */ 1243 for (i = 0; i < hw->queue_pairs; i++) { 1244 for (j = 0; j < NTB_XSTATS_NUM; j++) { 1245 off = j + (i + 1) * NTB_XSTATS_NUM; 1246 snprintf(xstats_names[off].name, 1247 sizeof(xstats_names[0].name), 1248 "%s_q%u", ntb_xstats_names[j].name, i); 1249 } 1250 } 1251 1252 return xstats_num; 1253 } 1254 1255 static uint64_t 1256 ntb_xstats_get_by_name(const struct rte_rawdev *dev, 1257 const char *name, unsigned int *id) 1258 { 1259 struct rte_rawdev_xstats_name *xstats_names; 1260 struct ntb_hw *hw = dev->dev_private; 1261 uint32_t xstats_num, i, j, off; 1262 1263 if (name == NULL) 1264 return -EINVAL; 1265 1266 xstats_num = NTB_XSTATS_NUM * (hw->queue_pairs + 1); 1267 xstats_names = rte_zmalloc("ntb_stats_name", 1268 sizeof(struct rte_rawdev_xstats_name) * 1269 xstats_num, 0); 1270 ntb_xstats_get_names(dev, xstats_names, xstats_num); 1271 1272 /* Calculate total stats of all queues. */ 1273 for (i = 0; i < NTB_XSTATS_NUM; i++) { 1274 for (j = 0; j < hw->queue_pairs; j++) { 1275 off = NTB_XSTATS_NUM * (j + 1) + i; 1276 hw->ntb_xstats[i] += 1277 ntb_stats_update(hw->ntb_xstats_off[off], 1278 hw->ntb_xstats[off]); 1279 } 1280 } 1281 1282 for (i = 0; i < xstats_num; i++) { 1283 if (!strncmp(name, xstats_names[i].name, 1284 RTE_RAW_DEV_XSTATS_NAME_SIZE)) { 1285 *id = i; 1286 rte_free(xstats_names); 1287 if (i < NTB_XSTATS_NUM) 1288 return hw->ntb_xstats[i]; 1289 else 1290 return ntb_stats_update(hw->ntb_xstats_off[i], 1291 hw->ntb_xstats[i]); 1292 } 1293 } 1294 1295 NTB_LOG(ERR, "Cannot find the xstats name."); 1296 1297 return -EINVAL; 1298 } 1299 1300 static int 1301 ntb_xstats_reset(struct rte_rawdev *dev, 1302 const uint32_t ids[], 1303 uint32_t nb_ids) 1304 { 1305 struct ntb_hw *hw = dev->dev_private; 1306 uint32_t i, j, off, xstats_num; 1307 1308 xstats_num = NTB_XSTATS_NUM * (hw->queue_pairs + 1); 1309 for (i = 0; i < nb_ids && ids[i] < xstats_num; i++) { 1310 if (ids[i] < NTB_XSTATS_NUM) { 1311 for (j = 0; j < hw->queue_pairs; j++) { 1312 off = NTB_XSTATS_NUM * (j + 1) + ids[i]; 1313 hw->ntb_xstats_off[off] = hw->ntb_xstats[off]; 1314 } 1315 } else { 1316 hw->ntb_xstats_off[ids[i]] = hw->ntb_xstats[ids[i]]; 1317 } 1318 } 1319 1320 return i; 1321 } 1322 1323 static const struct rte_rawdev_ops ntb_ops = { 1324 .dev_info_get = ntb_dev_info_get, 1325 .dev_configure = ntb_dev_configure, 1326 .dev_start = ntb_dev_start, 1327 .dev_stop = ntb_dev_stop, 1328 .dev_close = ntb_dev_close, 1329 .dev_reset = ntb_dev_reset, 1330 1331 .queue_def_conf = ntb_queue_conf_get, 1332 .queue_setup = ntb_queue_setup, 1333 .queue_release = ntb_queue_release, 1334 .queue_count = ntb_queue_count, 1335 1336 .enqueue_bufs = ntb_enqueue_bufs, 1337 .dequeue_bufs = ntb_dequeue_bufs, 1338 1339 .attr_get = ntb_attr_get, 1340 .attr_set = ntb_attr_set, 1341 1342 .xstats_get = ntb_xstats_get, 1343 .xstats_get_names = ntb_xstats_get_names, 1344 .xstats_get_by_name = ntb_xstats_get_by_name, 1345 .xstats_reset = ntb_xstats_reset, 1346 }; 1347 1348 static int 1349 ntb_init_hw(struct rte_rawdev *dev, struct rte_pci_device *pci_dev) 1350 { 1351 struct ntb_hw *hw = dev->dev_private; 1352 struct rte_intr_handle *intr_handle; 1353 int ret, i; 1354 1355 hw->pci_dev = pci_dev; 1356 hw->peer_dev_up = 0; 1357 hw->link_status = NTB_LINK_DOWN; 1358 hw->link_speed = NTB_SPEED_NONE; 1359 hw->link_width = NTB_WIDTH_NONE; 1360 1361 switch (pci_dev->id.device_id) { 1362 case NTB_INTEL_DEV_ID_B2B_SKX: 1363 hw->ntb_ops = &intel_ntb_ops; 1364 break; 1365 default: 1366 NTB_LOG(ERR, "Not supported device."); 1367 return -EINVAL; 1368 } 1369 1370 if (hw->ntb_ops->ntb_dev_init == NULL) 1371 return -ENOTSUP; 1372 ret = (*hw->ntb_ops->ntb_dev_init)(dev); 1373 if (ret) { 1374 NTB_LOG(ERR, "Unable to init ntb dev."); 1375 return ret; 1376 } 1377 1378 if (hw->ntb_ops->set_link == NULL) 1379 return -ENOTSUP; 1380 ret = (*hw->ntb_ops->set_link)(dev, 1); 1381 if (ret) 1382 return ret; 1383 1384 /* Init doorbell. */ 1385 hw->db_valid_mask = RTE_LEN2MASK(hw->db_cnt, uint64_t); 1386 1387 intr_handle = &pci_dev->intr_handle; 1388 /* Register callback func to eal lib */ 1389 rte_intr_callback_register(intr_handle, 1390 ntb_dev_intr_handler, dev); 1391 1392 ret = rte_intr_efd_enable(intr_handle, hw->db_cnt); 1393 if (ret) 1394 return ret; 1395 1396 /* To clarify, the interrupt for each doorbell is already mapped 1397 * by default for intel gen3. They are mapped to msix vec 1-32, 1398 * and hardware intr is mapped to 0. Map all to 0 for uio. 1399 */ 1400 if (!rte_intr_cap_multiple(intr_handle)) { 1401 for (i = 0; i < hw->db_cnt; i++) { 1402 if (hw->ntb_ops->vector_bind == NULL) 1403 return -ENOTSUP; 1404 ret = (*hw->ntb_ops->vector_bind)(dev, i, 0); 1405 if (ret) 1406 return ret; 1407 } 1408 } 1409 1410 if (hw->ntb_ops->db_set_mask == NULL || 1411 hw->ntb_ops->peer_db_set == NULL) { 1412 NTB_LOG(ERR, "Doorbell is not supported."); 1413 return -ENOTSUP; 1414 } 1415 hw->db_mask = 0; 1416 ret = (*hw->ntb_ops->db_set_mask)(dev, hw->db_mask); 1417 if (ret) { 1418 NTB_LOG(ERR, "Unable to enable intr for all dbs."); 1419 return ret; 1420 } 1421 1422 /* enable uio intr after callback register */ 1423 rte_intr_enable(intr_handle); 1424 1425 return ret; 1426 } 1427 1428 static int 1429 ntb_create(struct rte_pci_device *pci_dev, int socket_id) 1430 { 1431 char name[RTE_RAWDEV_NAME_MAX_LEN]; 1432 struct rte_rawdev *rawdev = NULL; 1433 int ret; 1434 1435 if (pci_dev == NULL) { 1436 NTB_LOG(ERR, "Invalid pci_dev."); 1437 return -EINVAL; 1438 } 1439 1440 memset(name, 0, sizeof(name)); 1441 snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "NTB:%x:%02x.%x", 1442 pci_dev->addr.bus, pci_dev->addr.devid, 1443 pci_dev->addr.function); 1444 1445 NTB_LOG(INFO, "Init %s on NUMA node %d", name, socket_id); 1446 1447 /* Allocate device structure. */ 1448 rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct ntb_hw), 1449 socket_id); 1450 if (rawdev == NULL) { 1451 NTB_LOG(ERR, "Unable to allocate rawdev."); 1452 return -EINVAL; 1453 } 1454 1455 rawdev->dev_ops = &ntb_ops; 1456 rawdev->device = &pci_dev->device; 1457 rawdev->driver_name = pci_dev->driver->driver.name; 1458 1459 ret = ntb_init_hw(rawdev, pci_dev); 1460 if (ret < 0) { 1461 NTB_LOG(ERR, "Unable to init ntb hw."); 1462 goto fail; 1463 } 1464 1465 return ret; 1466 1467 fail: 1468 if (rawdev != NULL) 1469 rte_rawdev_pmd_release(rawdev); 1470 1471 return ret; 1472 } 1473 1474 static int 1475 ntb_destroy(struct rte_pci_device *pci_dev) 1476 { 1477 char name[RTE_RAWDEV_NAME_MAX_LEN]; 1478 struct rte_rawdev *rawdev; 1479 int ret; 1480 1481 if (pci_dev == NULL) { 1482 NTB_LOG(ERR, "Invalid pci_dev."); 1483 ret = -EINVAL; 1484 return ret; 1485 } 1486 1487 memset(name, 0, sizeof(name)); 1488 snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "NTB:%x:%02x.%x", 1489 pci_dev->addr.bus, pci_dev->addr.devid, 1490 pci_dev->addr.function); 1491 1492 NTB_LOG(INFO, "Closing %s on NUMA node %d", name, rte_socket_id()); 1493 1494 rawdev = rte_rawdev_pmd_get_named_dev(name); 1495 if (rawdev == NULL) { 1496 NTB_LOG(ERR, "Invalid device name (%s)", name); 1497 ret = -EINVAL; 1498 return ret; 1499 } 1500 1501 ret = rte_rawdev_pmd_release(rawdev); 1502 if (ret) 1503 NTB_LOG(ERR, "Failed to destroy ntb rawdev."); 1504 1505 return ret; 1506 } 1507 1508 static int 1509 ntb_probe(struct rte_pci_driver *pci_drv __rte_unused, 1510 struct rte_pci_device *pci_dev) 1511 { 1512 return ntb_create(pci_dev, rte_socket_id()); 1513 } 1514 1515 static int 1516 ntb_remove(struct rte_pci_device *pci_dev) 1517 { 1518 return ntb_destroy(pci_dev); 1519 } 1520 1521 1522 static struct rte_pci_driver rte_ntb_pmd = { 1523 .id_table = pci_id_ntb_map, 1524 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_WC_ACTIVATE, 1525 .probe = ntb_probe, 1526 .remove = ntb_remove, 1527 }; 1528 1529 RTE_PMD_REGISTER_PCI(raw_ntb, rte_ntb_pmd); 1530 RTE_PMD_REGISTER_PCI_TABLE(raw_ntb, pci_id_ntb_map); 1531 RTE_PMD_REGISTER_KMOD_DEP(raw_ntb, "* igb_uio | uio_pci_generic | vfio-pci"); 1532 RTE_LOG_REGISTER(ntb_logtype, pmd.raw.ntb, INFO); 1533