1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2019 Intel Corporation 3 */ 4 #include <stdint.h> 5 #include <stdio.h> 6 #include <inttypes.h> 7 #include <unistd.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <getopt.h> 11 12 #include <cmdline_parse_string.h> 13 #include <cmdline_socket.h> 14 #include <cmdline.h> 15 #include <rte_common.h> 16 #include <rte_rawdev.h> 17 #include <rte_ethdev.h> 18 #include <rte_malloc.h> 19 #include <rte_lcore.h> 20 #include <rte_cycles.h> 21 #include <rte_pmd_ntb.h> 22 23 /* Per-port statistics struct */ 24 struct ntb_port_statistics { 25 uint64_t tx; 26 uint64_t rx; 27 } __rte_cache_aligned; 28 /* Port 0: NTB dev, Port 1: ethdev when iofwd. */ 29 struct ntb_port_statistics ntb_port_stats[2]; 30 31 struct ntb_fwd_stream { 32 uint16_t tx_port; 33 uint16_t rx_port; 34 uint16_t qp_id; 35 uint8_t tx_ntb; /* If ntb device is tx port. */ 36 }; 37 38 struct ntb_fwd_lcore_conf { 39 uint16_t stream_id; 40 uint16_t nb_stream; 41 uint8_t stopped; 42 }; 43 44 enum ntb_fwd_mode { 45 FILE_TRANS = 0, 46 RXONLY, 47 TXONLY, 48 IOFWD, 49 MAX_FWD_MODE, 50 }; 51 static const char *const fwd_mode_s[] = { 52 "file-trans", 53 "rxonly", 54 "txonly", 55 "iofwd", 56 NULL, 57 }; 58 static enum ntb_fwd_mode fwd_mode = MAX_FWD_MODE; 59 60 static struct ntb_fwd_lcore_conf fwd_lcore_conf[RTE_MAX_LCORE]; 61 static struct ntb_fwd_stream *fwd_streams; 62 63 static struct rte_mempool *mbuf_pool; 64 65 #define NTB_DRV_NAME_LEN 7 66 #define MEMPOOL_CACHE_SIZE 256 67 68 static uint8_t in_test; 69 static uint8_t interactive = 1; 70 static uint16_t eth_port_id = RTE_MAX_ETHPORTS; 71 static uint16_t dev_id; 72 73 /* Number of queues, default set as 1 */ 74 static uint16_t num_queues = 1; 75 static uint16_t ntb_buf_size = RTE_MBUF_DEFAULT_BUF_SIZE; 76 77 /* Configurable number of descriptors */ 78 #define NTB_DEFAULT_NUM_DESCS 1024 79 static uint16_t nb_desc = NTB_DEFAULT_NUM_DESCS; 80 81 static uint16_t tx_free_thresh; 82 83 #define NTB_MAX_PKT_BURST 32 84 #define NTB_DFLT_PKT_BURST 32 85 static uint16_t pkt_burst = NTB_DFLT_PKT_BURST; 86 87 #define BURST_TX_RETRIES 64 88 89 static struct rte_eth_conf eth_port_conf = { 90 .rxmode = { 91 .mq_mode = ETH_MQ_RX_RSS, 92 .split_hdr_size = 0, 93 }, 94 .rx_adv_conf = { 95 .rss_conf = { 96 .rss_key = NULL, 97 .rss_hf = ETH_RSS_IP, 98 }, 99 }, 100 .txmode = { 101 .mq_mode = ETH_MQ_TX_NONE, 102 }, 103 }; 104 105 /* *** Help command with introduction. *** */ 106 struct cmd_help_result { 107 cmdline_fixed_string_t help; 108 }; 109 110 static void 111 cmd_help_parsed(__attribute__((unused)) void *parsed_result, 112 struct cmdline *cl, 113 __attribute__((unused)) void *data) 114 { 115 cmdline_printf( 116 cl, 117 "\n" 118 "The following commands are currently available:\n\n" 119 "Control:\n" 120 " quit :" 121 " Quit the application.\n" 122 "\nTransmission:\n" 123 " send [path] :" 124 " Send [path] file. Only take effect in file-trans mode\n" 125 " start :" 126 " Start transmissions.\n" 127 " stop :" 128 " Stop transmissions.\n" 129 " clear/show port stats :" 130 " Clear/show port stats.\n" 131 " set fwd file-trans/rxonly/txonly/iofwd :" 132 " Set packet forwarding mode.\n" 133 ); 134 135 } 136 137 cmdline_parse_token_string_t cmd_help_help = 138 TOKEN_STRING_INITIALIZER(struct cmd_help_result, help, "help"); 139 140 cmdline_parse_inst_t cmd_help = { 141 .f = cmd_help_parsed, 142 .data = NULL, 143 .help_str = "show help", 144 .tokens = { 145 (void *)&cmd_help_help, 146 NULL, 147 }, 148 }; 149 150 /* *** QUIT *** */ 151 struct cmd_quit_result { 152 cmdline_fixed_string_t quit; 153 }; 154 155 static void 156 cmd_quit_parsed(__attribute__((unused)) void *parsed_result, 157 struct cmdline *cl, 158 __attribute__((unused)) void *data) 159 { 160 struct ntb_fwd_lcore_conf *conf; 161 uint8_t lcore_id; 162 163 /* Stop transmission first. */ 164 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 165 conf = &fwd_lcore_conf[lcore_id]; 166 167 if (!conf->nb_stream) 168 continue; 169 170 if (conf->stopped) 171 continue; 172 173 conf->stopped = 1; 174 } 175 printf("\nWaiting for lcores to finish...\n"); 176 rte_eal_mp_wait_lcore(); 177 in_test = 0; 178 179 /* Stop traffic and Close port. */ 180 rte_rawdev_stop(dev_id); 181 rte_rawdev_close(dev_id); 182 if (eth_port_id < RTE_MAX_ETHPORTS && fwd_mode == IOFWD) { 183 rte_eth_dev_stop(eth_port_id); 184 rte_eth_dev_close(eth_port_id); 185 } 186 187 cmdline_quit(cl); 188 } 189 190 cmdline_parse_token_string_t cmd_quit_quit = 191 TOKEN_STRING_INITIALIZER(struct cmd_quit_result, quit, "quit"); 192 193 cmdline_parse_inst_t cmd_quit = { 194 .f = cmd_quit_parsed, 195 .data = NULL, 196 .help_str = "exit application", 197 .tokens = { 198 (void *)&cmd_quit_quit, 199 NULL, 200 }, 201 }; 202 203 /* *** SEND FILE PARAMETERS *** */ 204 struct cmd_sendfile_result { 205 cmdline_fixed_string_t send_string; 206 char filepath[]; 207 }; 208 209 static void 210 cmd_sendfile_parsed(void *parsed_result, 211 __attribute__((unused)) struct cmdline *cl, 212 __attribute__((unused)) void *data) 213 { 214 struct cmd_sendfile_result *res = parsed_result; 215 struct rte_rawdev_buf *pkts_send[NTB_MAX_PKT_BURST]; 216 struct rte_mbuf *mbuf_send[NTB_MAX_PKT_BURST]; 217 uint64_t size, count, i, nb_burst; 218 uint16_t nb_tx, buf_size; 219 unsigned int nb_pkt; 220 size_t queue_id = 0; 221 uint16_t retry = 0; 222 uint32_t val; 223 FILE *file; 224 225 if (num_queues != 1) { 226 printf("File transmission only supports 1 queue.\n"); 227 num_queues = 1; 228 } 229 230 file = fopen(res->filepath, "r"); 231 if (file == NULL) { 232 printf("Fail to open the file.\n"); 233 return; 234 } 235 236 if (fseek(file, 0, SEEK_END) < 0) { 237 printf("Fail to get file size.\n"); 238 fclose(file); 239 return; 240 } 241 size = ftell(file); 242 if (fseek(file, 0, SEEK_SET) < 0) { 243 printf("Fail to get file size.\n"); 244 fclose(file); 245 return; 246 } 247 248 /* Tell remote about the file size. */ 249 val = size >> 32; 250 rte_rawdev_set_attr(dev_id, "spad_user_0", val); 251 val = size; 252 rte_rawdev_set_attr(dev_id, "spad_user_1", val); 253 printf("Sending file, size is %"PRIu64"\n", size); 254 255 for (i = 0; i < NTB_MAX_PKT_BURST; i++) 256 pkts_send[i] = (struct rte_rawdev_buf *) 257 malloc(sizeof(struct rte_rawdev_buf)); 258 259 buf_size = ntb_buf_size - RTE_PKTMBUF_HEADROOM; 260 count = (size + buf_size - 1) / buf_size; 261 nb_burst = (count + pkt_burst - 1) / pkt_burst; 262 263 for (i = 0; i < nb_burst; i++) { 264 val = RTE_MIN(count, pkt_burst); 265 if (rte_mempool_get_bulk(mbuf_pool, (void **)mbuf_send, 266 val) == 0) { 267 for (nb_pkt = 0; nb_pkt < val; nb_pkt++) { 268 mbuf_send[nb_pkt]->port = dev_id; 269 mbuf_send[nb_pkt]->data_len = 270 fread(rte_pktmbuf_mtod(mbuf_send[nb_pkt], 271 void *), 1, buf_size, file); 272 mbuf_send[nb_pkt]->pkt_len = 273 mbuf_send[nb_pkt]->data_len; 274 pkts_send[nb_pkt]->buf_addr = mbuf_send[nb_pkt]; 275 } 276 } else { 277 for (nb_pkt = 0; nb_pkt < val; nb_pkt++) { 278 mbuf_send[nb_pkt] = 279 rte_mbuf_raw_alloc(mbuf_pool); 280 if (mbuf_send[nb_pkt] == NULL) 281 break; 282 mbuf_send[nb_pkt]->port = dev_id; 283 mbuf_send[nb_pkt]->data_len = 284 fread(rte_pktmbuf_mtod(mbuf_send[nb_pkt], 285 void *), 1, buf_size, file); 286 mbuf_send[nb_pkt]->pkt_len = 287 mbuf_send[nb_pkt]->data_len; 288 pkts_send[nb_pkt]->buf_addr = mbuf_send[nb_pkt]; 289 } 290 } 291 292 nb_tx = rte_rawdev_enqueue_buffers(dev_id, pkts_send, nb_pkt, 293 (void *)queue_id); 294 while (nb_tx != nb_pkt && retry < BURST_TX_RETRIES) { 295 rte_delay_us(1); 296 nb_tx += rte_rawdev_enqueue_buffers(dev_id, 297 &pkts_send[nb_tx], nb_pkt - nb_tx, 298 (void *)queue_id); 299 } 300 count -= nb_pkt; 301 } 302 /* Clear register after file sending done. */ 303 rte_rawdev_set_attr(dev_id, "spad_user_0", 0); 304 rte_rawdev_set_attr(dev_id, "spad_user_1", 0); 305 printf("Done sending file.\n"); 306 307 for (i = 0; i < NTB_MAX_PKT_BURST; i++) 308 free(pkts_send[i]); 309 fclose(file); 310 } 311 312 cmdline_parse_token_string_t cmd_send_file_send = 313 TOKEN_STRING_INITIALIZER(struct cmd_sendfile_result, send_string, 314 "send"); 315 cmdline_parse_token_string_t cmd_send_file_filepath = 316 TOKEN_STRING_INITIALIZER(struct cmd_sendfile_result, filepath, NULL); 317 318 319 cmdline_parse_inst_t cmd_send_file = { 320 .f = cmd_sendfile_parsed, 321 .data = NULL, 322 .help_str = "send <file_path>", 323 .tokens = { 324 (void *)&cmd_send_file_send, 325 (void *)&cmd_send_file_filepath, 326 NULL, 327 }, 328 }; 329 330 #define RECV_FILE_LEN 30 331 static int 332 start_polling_recv_file(void *param) 333 { 334 struct rte_rawdev_buf *pkts_recv[NTB_MAX_PKT_BURST]; 335 struct ntb_fwd_lcore_conf *conf = param; 336 struct rte_mbuf *mbuf; 337 char filepath[RECV_FILE_LEN]; 338 uint64_t val, size, file_len; 339 uint16_t nb_rx, i, file_no; 340 size_t queue_id = 0; 341 FILE *file; 342 343 for (i = 0; i < NTB_MAX_PKT_BURST; i++) 344 pkts_recv[i] = (struct rte_rawdev_buf *) 345 malloc(sizeof(struct rte_rawdev_buf)); 346 347 file_no = 0; 348 while (!conf->stopped) { 349 snprintf(filepath, RECV_FILE_LEN, "ntb_recv_file%d", file_no); 350 file = fopen(filepath, "w"); 351 if (file == NULL) { 352 printf("Fail to open the file.\n"); 353 return -EINVAL; 354 } 355 356 rte_rawdev_get_attr(dev_id, "spad_user_0", &val); 357 size = val << 32; 358 rte_rawdev_get_attr(dev_id, "spad_user_1", &val); 359 size |= val; 360 361 if (!size) { 362 fclose(file); 363 continue; 364 } 365 366 file_len = 0; 367 nb_rx = NTB_MAX_PKT_BURST; 368 while (file_len < size && !conf->stopped) { 369 nb_rx = rte_rawdev_dequeue_buffers(dev_id, pkts_recv, 370 pkt_burst, (void *)queue_id); 371 ntb_port_stats[0].rx += nb_rx; 372 for (i = 0; i < nb_rx; i++) { 373 mbuf = pkts_recv[i]->buf_addr; 374 fwrite(rte_pktmbuf_mtod(mbuf, void *), 1, 375 mbuf->data_len, file); 376 file_len += mbuf->data_len; 377 rte_pktmbuf_free(mbuf); 378 pkts_recv[i]->buf_addr = NULL; 379 } 380 } 381 382 printf("Received file (size: %" PRIu64 ") from peer to %s.\n", 383 size, filepath); 384 fclose(file); 385 file_no++; 386 } 387 388 for (i = 0; i < NTB_MAX_PKT_BURST; i++) 389 free(pkts_recv[i]); 390 return 0; 391 } 392 393 static int 394 start_iofwd_per_lcore(void *param) 395 { 396 struct rte_rawdev_buf *ntb_buf[NTB_MAX_PKT_BURST]; 397 struct rte_mbuf *pkts_burst[NTB_MAX_PKT_BURST]; 398 struct ntb_fwd_lcore_conf *conf = param; 399 struct ntb_fwd_stream fs; 400 uint16_t nb_rx, nb_tx; 401 int i, j; 402 403 for (i = 0; i < NTB_MAX_PKT_BURST; i++) 404 ntb_buf[i] = (struct rte_rawdev_buf *) 405 malloc(sizeof(struct rte_rawdev_buf)); 406 407 while (!conf->stopped) { 408 for (i = 0; i < conf->nb_stream; i++) { 409 fs = fwd_streams[conf->stream_id + i]; 410 if (fs.tx_ntb) { 411 nb_rx = rte_eth_rx_burst(fs.rx_port, 412 fs.qp_id, pkts_burst, 413 pkt_burst); 414 if (unlikely(nb_rx == 0)) 415 continue; 416 for (j = 0; j < nb_rx; j++) 417 ntb_buf[j]->buf_addr = pkts_burst[j]; 418 nb_tx = 419 rte_rawdev_enqueue_buffers(fs.tx_port, 420 ntb_buf, nb_rx, 421 (void *)(size_t)fs.qp_id); 422 ntb_port_stats[0].tx += nb_tx; 423 ntb_port_stats[1].rx += nb_rx; 424 } else { 425 nb_rx = 426 rte_rawdev_dequeue_buffers(fs.rx_port, 427 ntb_buf, pkt_burst, 428 (void *)(size_t)fs.qp_id); 429 if (unlikely(nb_rx == 0)) 430 continue; 431 for (j = 0; j < nb_rx; j++) 432 pkts_burst[j] = ntb_buf[j]->buf_addr; 433 nb_tx = rte_eth_tx_burst(fs.tx_port, 434 fs.qp_id, pkts_burst, nb_rx); 435 ntb_port_stats[1].tx += nb_tx; 436 ntb_port_stats[0].rx += nb_rx; 437 } 438 if (unlikely(nb_tx < nb_rx)) { 439 do { 440 rte_pktmbuf_free(pkts_burst[nb_tx]); 441 } while (++nb_tx < nb_rx); 442 } 443 } 444 } 445 446 for (i = 0; i < NTB_MAX_PKT_BURST; i++) 447 free(ntb_buf[i]); 448 449 return 0; 450 } 451 452 static int 453 start_rxonly_per_lcore(void *param) 454 { 455 struct rte_rawdev_buf *ntb_buf[NTB_MAX_PKT_BURST]; 456 struct ntb_fwd_lcore_conf *conf = param; 457 struct ntb_fwd_stream fs; 458 uint16_t nb_rx; 459 int i, j; 460 461 for (i = 0; i < NTB_MAX_PKT_BURST; i++) 462 ntb_buf[i] = (struct rte_rawdev_buf *) 463 malloc(sizeof(struct rte_rawdev_buf)); 464 465 while (!conf->stopped) { 466 for (i = 0; i < conf->nb_stream; i++) { 467 fs = fwd_streams[conf->stream_id + i]; 468 nb_rx = rte_rawdev_dequeue_buffers(fs.rx_port, 469 ntb_buf, pkt_burst, (void *)(size_t)fs.qp_id); 470 if (unlikely(nb_rx == 0)) 471 continue; 472 ntb_port_stats[0].rx += nb_rx; 473 474 for (j = 0; j < nb_rx; j++) 475 rte_pktmbuf_free(ntb_buf[j]->buf_addr); 476 } 477 } 478 479 for (i = 0; i < NTB_MAX_PKT_BURST; i++) 480 free(ntb_buf[i]); 481 482 return 0; 483 } 484 485 486 static int 487 start_txonly_per_lcore(void *param) 488 { 489 struct rte_rawdev_buf *ntb_buf[NTB_MAX_PKT_BURST]; 490 struct rte_mbuf *pkts_burst[NTB_MAX_PKT_BURST]; 491 struct ntb_fwd_lcore_conf *conf = param; 492 struct ntb_fwd_stream fs; 493 uint16_t nb_pkt, nb_tx; 494 int i; 495 496 for (i = 0; i < NTB_MAX_PKT_BURST; i++) 497 ntb_buf[i] = (struct rte_rawdev_buf *) 498 malloc(sizeof(struct rte_rawdev_buf)); 499 500 while (!conf->stopped) { 501 for (i = 0; i < conf->nb_stream; i++) { 502 fs = fwd_streams[conf->stream_id + i]; 503 if (rte_mempool_get_bulk(mbuf_pool, (void **)pkts_burst, 504 pkt_burst) == 0) { 505 for (nb_pkt = 0; nb_pkt < pkt_burst; nb_pkt++) { 506 pkts_burst[nb_pkt]->port = dev_id; 507 pkts_burst[nb_pkt]->data_len = 508 pkts_burst[nb_pkt]->buf_len - 509 RTE_PKTMBUF_HEADROOM; 510 pkts_burst[nb_pkt]->pkt_len = 511 pkts_burst[nb_pkt]->data_len; 512 ntb_buf[nb_pkt]->buf_addr = 513 pkts_burst[nb_pkt]; 514 } 515 } else { 516 for (nb_pkt = 0; nb_pkt < pkt_burst; nb_pkt++) { 517 pkts_burst[nb_pkt] = 518 rte_pktmbuf_alloc(mbuf_pool); 519 if (pkts_burst[nb_pkt] == NULL) 520 break; 521 pkts_burst[nb_pkt]->port = dev_id; 522 pkts_burst[nb_pkt]->data_len = 523 pkts_burst[nb_pkt]->buf_len - 524 RTE_PKTMBUF_HEADROOM; 525 pkts_burst[nb_pkt]->pkt_len = 526 pkts_burst[nb_pkt]->data_len; 527 ntb_buf[nb_pkt]->buf_addr = 528 pkts_burst[nb_pkt]; 529 } 530 } 531 nb_tx = rte_rawdev_enqueue_buffers(fs.tx_port, 532 ntb_buf, nb_pkt, (void *)(size_t)fs.qp_id); 533 ntb_port_stats[0].tx += nb_tx; 534 if (unlikely(nb_tx < nb_pkt)) { 535 do { 536 rte_pktmbuf_free( 537 ntb_buf[nb_tx]->buf_addr); 538 } while (++nb_tx < nb_pkt); 539 } 540 } 541 } 542 543 for (i = 0; i < NTB_MAX_PKT_BURST; i++) 544 free(ntb_buf[i]); 545 546 return 0; 547 } 548 549 static int 550 ntb_fwd_config_setup(void) 551 { 552 uint16_t i; 553 554 /* Make sure iofwd has valid ethdev. */ 555 if (fwd_mode == IOFWD && eth_port_id >= RTE_MAX_ETHPORTS) { 556 printf("No ethdev, cannot be in iofwd mode."); 557 return -EINVAL; 558 } 559 560 if (fwd_mode == IOFWD) { 561 fwd_streams = rte_zmalloc("ntb_fwd: fwd_streams", 562 sizeof(struct ntb_fwd_stream) * num_queues * 2, 563 RTE_CACHE_LINE_SIZE); 564 for (i = 0; i < num_queues; i++) { 565 fwd_streams[i * 2].qp_id = i; 566 fwd_streams[i * 2].tx_port = dev_id; 567 fwd_streams[i * 2].rx_port = eth_port_id; 568 fwd_streams[i * 2].tx_ntb = 1; 569 570 fwd_streams[i * 2 + 1].qp_id = i; 571 fwd_streams[i * 2 + 1].tx_port = eth_port_id; 572 fwd_streams[i * 2 + 1].rx_port = dev_id; 573 fwd_streams[i * 2 + 1].tx_ntb = 0; 574 } 575 return 0; 576 } 577 578 if (fwd_mode == RXONLY || fwd_mode == FILE_TRANS) { 579 /* Only support 1 queue in file-trans for in order. */ 580 if (fwd_mode == FILE_TRANS) 581 num_queues = 1; 582 583 fwd_streams = rte_zmalloc("ntb_fwd: fwd_streams", 584 sizeof(struct ntb_fwd_stream) * num_queues, 585 RTE_CACHE_LINE_SIZE); 586 for (i = 0; i < num_queues; i++) { 587 fwd_streams[i].qp_id = i; 588 fwd_streams[i].tx_port = RTE_MAX_ETHPORTS; 589 fwd_streams[i].rx_port = dev_id; 590 fwd_streams[i].tx_ntb = 0; 591 } 592 return 0; 593 } 594 595 if (fwd_mode == TXONLY) { 596 fwd_streams = rte_zmalloc("ntb_fwd: fwd_streams", 597 sizeof(struct ntb_fwd_stream) * num_queues, 598 RTE_CACHE_LINE_SIZE); 599 for (i = 0; i < num_queues; i++) { 600 fwd_streams[i].qp_id = i; 601 fwd_streams[i].tx_port = dev_id; 602 fwd_streams[i].rx_port = RTE_MAX_ETHPORTS; 603 fwd_streams[i].tx_ntb = 1; 604 } 605 } 606 return 0; 607 } 608 609 static void 610 assign_stream_to_lcores(void) 611 { 612 struct ntb_fwd_lcore_conf *conf; 613 struct ntb_fwd_stream *fs; 614 uint16_t nb_streams, sm_per_lcore, sm_id, i; 615 uint8_t lcore_id, lcore_num, nb_extra; 616 617 lcore_num = rte_lcore_count(); 618 /* Exclude master core */ 619 lcore_num--; 620 621 nb_streams = (fwd_mode == IOFWD) ? num_queues * 2 : num_queues; 622 623 sm_per_lcore = nb_streams / lcore_num; 624 nb_extra = nb_streams % lcore_num; 625 sm_id = 0; 626 i = 0; 627 628 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 629 conf = &fwd_lcore_conf[lcore_id]; 630 631 if (i < nb_extra) { 632 conf->nb_stream = sm_per_lcore + 1; 633 conf->stream_id = sm_id; 634 sm_id = sm_id + sm_per_lcore + 1; 635 } else { 636 conf->nb_stream = sm_per_lcore; 637 conf->stream_id = sm_id; 638 sm_id = sm_id + sm_per_lcore; 639 } 640 641 i++; 642 if (sm_id >= nb_streams) 643 break; 644 } 645 646 /* Print packet forwading config. */ 647 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 648 conf = &fwd_lcore_conf[lcore_id]; 649 650 if (!conf->nb_stream) 651 continue; 652 653 printf("Streams on Lcore %u :\n", lcore_id); 654 for (i = 0; i < conf->nb_stream; i++) { 655 fs = &fwd_streams[conf->stream_id + i]; 656 if (fwd_mode == IOFWD) 657 printf(" + Stream %u : %s%u RX -> %s%u TX," 658 " Q=%u\n", conf->stream_id + i, 659 fs->tx_ntb ? "Eth" : "NTB", fs->rx_port, 660 fs->tx_ntb ? "NTB" : "Eth", fs->tx_port, 661 fs->qp_id); 662 if (fwd_mode == FILE_TRANS || fwd_mode == RXONLY) 663 printf(" + Stream %u : %s%u RX only\n", 664 conf->stream_id, "NTB", fs->rx_port); 665 if (fwd_mode == TXONLY) 666 printf(" + Stream %u : %s%u TX only\n", 667 conf->stream_id, "NTB", fs->tx_port); 668 } 669 } 670 } 671 672 static void 673 start_pkt_fwd(void) 674 { 675 struct ntb_fwd_lcore_conf *conf; 676 struct rte_eth_link eth_link; 677 uint8_t lcore_id; 678 int ret, i; 679 680 ret = ntb_fwd_config_setup(); 681 if (ret < 0) { 682 printf("Cannot start traffic. Please reset fwd mode.\n"); 683 return; 684 } 685 686 /* If using iofwd, checking ethdev link status first. */ 687 if (fwd_mode == IOFWD) { 688 printf("Checking eth link status...\n"); 689 /* Wait for eth link up at most 100 times. */ 690 for (i = 0; i < 100; i++) { 691 rte_eth_link_get(eth_port_id, ð_link); 692 if (eth_link.link_status) { 693 printf("Eth%u Link Up. Speed %u Mbps - %s\n", 694 eth_port_id, eth_link.link_speed, 695 (eth_link.link_duplex == 696 ETH_LINK_FULL_DUPLEX) ? 697 ("full-duplex") : ("half-duplex")); 698 break; 699 } 700 } 701 if (!eth_link.link_status) { 702 printf("Eth%u link down. Cannot start traffic.\n", 703 eth_port_id); 704 return; 705 } 706 } 707 708 assign_stream_to_lcores(); 709 in_test = 1; 710 711 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 712 conf = &fwd_lcore_conf[lcore_id]; 713 714 if (!conf->nb_stream) 715 continue; 716 717 conf->stopped = 0; 718 if (fwd_mode == FILE_TRANS) 719 rte_eal_remote_launch(start_polling_recv_file, 720 conf, lcore_id); 721 else if (fwd_mode == IOFWD) 722 rte_eal_remote_launch(start_iofwd_per_lcore, 723 conf, lcore_id); 724 else if (fwd_mode == RXONLY) 725 rte_eal_remote_launch(start_rxonly_per_lcore, 726 conf, lcore_id); 727 else if (fwd_mode == TXONLY) 728 rte_eal_remote_launch(start_txonly_per_lcore, 729 conf, lcore_id); 730 } 731 } 732 733 /* *** START FWD PARAMETERS *** */ 734 struct cmd_start_result { 735 cmdline_fixed_string_t start; 736 }; 737 738 static void 739 cmd_start_parsed(__attribute__((unused)) void *parsed_result, 740 __attribute__((unused)) struct cmdline *cl, 741 __attribute__((unused)) void *data) 742 { 743 start_pkt_fwd(); 744 } 745 746 cmdline_parse_token_string_t cmd_start_start = 747 TOKEN_STRING_INITIALIZER(struct cmd_start_result, start, "start"); 748 749 cmdline_parse_inst_t cmd_start = { 750 .f = cmd_start_parsed, 751 .data = NULL, 752 .help_str = "start pkt fwd between ntb and ethdev", 753 .tokens = { 754 (void *)&cmd_start_start, 755 NULL, 756 }, 757 }; 758 759 /* *** STOP *** */ 760 struct cmd_stop_result { 761 cmdline_fixed_string_t stop; 762 }; 763 764 static void 765 cmd_stop_parsed(__attribute__((unused)) void *parsed_result, 766 __attribute__((unused)) struct cmdline *cl, 767 __attribute__((unused)) void *data) 768 { 769 struct ntb_fwd_lcore_conf *conf; 770 uint8_t lcore_id; 771 772 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 773 conf = &fwd_lcore_conf[lcore_id]; 774 775 if (!conf->nb_stream) 776 continue; 777 778 if (conf->stopped) 779 continue; 780 781 conf->stopped = 1; 782 } 783 printf("\nWaiting for lcores to finish...\n"); 784 rte_eal_mp_wait_lcore(); 785 in_test = 0; 786 printf("\nDone.\n"); 787 } 788 789 cmdline_parse_token_string_t cmd_stop_stop = 790 TOKEN_STRING_INITIALIZER(struct cmd_stop_result, stop, "stop"); 791 792 cmdline_parse_inst_t cmd_stop = { 793 .f = cmd_stop_parsed, 794 .data = NULL, 795 .help_str = "stop: Stop packet forwarding", 796 .tokens = { 797 (void *)&cmd_stop_stop, 798 NULL, 799 }, 800 }; 801 802 static void 803 ntb_stats_clear(void) 804 { 805 int nb_ids, i; 806 uint32_t *ids; 807 808 /* Clear NTB dev stats */ 809 nb_ids = rte_rawdev_xstats_names_get(dev_id, NULL, 0); 810 if (nb_ids < 0) { 811 printf("Error: Cannot get count of xstats\n"); 812 return; 813 } 814 ids = malloc(sizeof(uint32_t) * nb_ids); 815 for (i = 0; i < nb_ids; i++) 816 ids[i] = i; 817 rte_rawdev_xstats_reset(dev_id, ids, nb_ids); 818 printf("\n statistics for NTB port %d cleared\n", dev_id); 819 820 /* Clear Ethdev stats if have any */ 821 if (fwd_mode == IOFWD && eth_port_id != RTE_MAX_ETHPORTS) { 822 rte_eth_stats_reset(eth_port_id); 823 printf("\n statistics for ETH port %d cleared\n", eth_port_id); 824 } 825 } 826 827 static inline void 828 ntb_calculate_throughput(uint16_t port) { 829 uint64_t diff_pkts_rx, diff_pkts_tx, diff_cycles; 830 uint64_t mpps_rx, mpps_tx; 831 static uint64_t prev_pkts_rx[2]; 832 static uint64_t prev_pkts_tx[2]; 833 static uint64_t prev_cycles[2]; 834 835 diff_cycles = prev_cycles[port]; 836 prev_cycles[port] = rte_rdtsc(); 837 if (diff_cycles > 0) 838 diff_cycles = prev_cycles[port] - diff_cycles; 839 diff_pkts_rx = (ntb_port_stats[port].rx > prev_pkts_rx[port]) ? 840 (ntb_port_stats[port].rx - prev_pkts_rx[port]) : 0; 841 diff_pkts_tx = (ntb_port_stats[port].tx > prev_pkts_tx[port]) ? 842 (ntb_port_stats[port].tx - prev_pkts_tx[port]) : 0; 843 prev_pkts_rx[port] = ntb_port_stats[port].rx; 844 prev_pkts_tx[port] = ntb_port_stats[port].tx; 845 mpps_rx = diff_cycles > 0 ? 846 diff_pkts_rx * rte_get_tsc_hz() / diff_cycles : 0; 847 mpps_tx = diff_cycles > 0 ? 848 diff_pkts_tx * rte_get_tsc_hz() / diff_cycles : 0; 849 printf(" Throughput (since last show)\n"); 850 printf(" Rx-pps: %12"PRIu64"\n Tx-pps: %12"PRIu64"\n", 851 mpps_rx, mpps_tx); 852 853 } 854 855 static void 856 ntb_stats_display(void) 857 { 858 struct rte_rawdev_xstats_name *xstats_names; 859 struct rte_eth_stats stats; 860 uint64_t *values; 861 uint32_t *ids; 862 int nb_ids, i; 863 864 printf("###### statistics for NTB port %d #######\n", dev_id); 865 866 /* Get NTB dev stats and stats names */ 867 nb_ids = rte_rawdev_xstats_names_get(dev_id, NULL, 0); 868 if (nb_ids < 0) { 869 printf("Error: Cannot get count of xstats\n"); 870 return; 871 } 872 xstats_names = malloc(sizeof(struct rte_rawdev_xstats_name) * nb_ids); 873 if (xstats_names == NULL) { 874 printf("Cannot allocate memory for xstats lookup\n"); 875 return; 876 } 877 if (nb_ids != rte_rawdev_xstats_names_get( 878 dev_id, xstats_names, nb_ids)) { 879 printf("Error: Cannot get xstats lookup\n"); 880 free(xstats_names); 881 return; 882 } 883 ids = malloc(sizeof(uint32_t) * nb_ids); 884 for (i = 0; i < nb_ids; i++) 885 ids[i] = i; 886 values = malloc(sizeof(uint64_t) * nb_ids); 887 if (nb_ids != rte_rawdev_xstats_get(dev_id, ids, values, nb_ids)) { 888 printf("Error: Unable to get xstats\n"); 889 free(xstats_names); 890 free(values); 891 free(ids); 892 return; 893 } 894 895 /* Display NTB dev stats */ 896 for (i = 0; i < nb_ids; i++) 897 printf(" %s: %"PRIu64"\n", xstats_names[i].name, values[i]); 898 ntb_calculate_throughput(0); 899 900 /* Get Ethdev stats if have any */ 901 if (fwd_mode == IOFWD && eth_port_id != RTE_MAX_ETHPORTS) { 902 printf("###### statistics for ETH port %d ######\n", 903 eth_port_id); 904 rte_eth_stats_get(eth_port_id, &stats); 905 printf(" RX-packets: %"PRIu64"\n", stats.ipackets); 906 printf(" RX-bytes: %"PRIu64"\n", stats.ibytes); 907 printf(" RX-errors: %"PRIu64"\n", stats.ierrors); 908 printf(" RX-missed: %"PRIu64"\n", stats.imissed); 909 printf(" TX-packets: %"PRIu64"\n", stats.opackets); 910 printf(" TX-bytes: %"PRIu64"\n", stats.obytes); 911 printf(" TX-errors: %"PRIu64"\n", stats.oerrors); 912 ntb_calculate_throughput(1); 913 } 914 915 free(xstats_names); 916 free(values); 917 free(ids); 918 } 919 920 /* *** SHOW/CLEAR PORT STATS *** */ 921 struct cmd_stats_result { 922 cmdline_fixed_string_t show; 923 cmdline_fixed_string_t port; 924 cmdline_fixed_string_t stats; 925 }; 926 927 static void 928 cmd_stats_parsed(void *parsed_result, 929 __attribute__((unused)) struct cmdline *cl, 930 __attribute__((unused)) void *data) 931 { 932 struct cmd_stats_result *res = parsed_result; 933 if (!strcmp(res->show, "clear")) 934 ntb_stats_clear(); 935 else 936 ntb_stats_display(); 937 } 938 939 cmdline_parse_token_string_t cmd_stats_show = 940 TOKEN_STRING_INITIALIZER(struct cmd_stats_result, show, "show#clear"); 941 cmdline_parse_token_string_t cmd_stats_port = 942 TOKEN_STRING_INITIALIZER(struct cmd_stats_result, port, "port"); 943 cmdline_parse_token_string_t cmd_stats_stats = 944 TOKEN_STRING_INITIALIZER(struct cmd_stats_result, stats, "stats"); 945 946 947 cmdline_parse_inst_t cmd_stats = { 948 .f = cmd_stats_parsed, 949 .data = NULL, 950 .help_str = "show|clear port stats", 951 .tokens = { 952 (void *)&cmd_stats_show, 953 (void *)&cmd_stats_port, 954 (void *)&cmd_stats_stats, 955 NULL, 956 }, 957 }; 958 959 /* *** SET FORWARDING MODE *** */ 960 struct cmd_set_fwd_mode_result { 961 cmdline_fixed_string_t set; 962 cmdline_fixed_string_t fwd; 963 cmdline_fixed_string_t mode; 964 }; 965 966 static void 967 cmd_set_fwd_mode_parsed(__attribute__((unused)) void *parsed_result, 968 __attribute__((unused)) struct cmdline *cl, 969 __attribute__((unused)) void *data) 970 { 971 struct cmd_set_fwd_mode_result *res = parsed_result; 972 int i; 973 974 if (in_test) { 975 printf("Please stop traffic first.\n"); 976 return; 977 } 978 979 for (i = 0; i < MAX_FWD_MODE; i++) { 980 if (!strcmp(res->mode, fwd_mode_s[i])) { 981 fwd_mode = i; 982 return; 983 } 984 } 985 printf("Invalid %s packet forwarding mode.\n", res->mode); 986 } 987 988 cmdline_parse_token_string_t cmd_setfwd_set = 989 TOKEN_STRING_INITIALIZER(struct cmd_set_fwd_mode_result, set, "set"); 990 cmdline_parse_token_string_t cmd_setfwd_fwd = 991 TOKEN_STRING_INITIALIZER(struct cmd_set_fwd_mode_result, fwd, "fwd"); 992 cmdline_parse_token_string_t cmd_setfwd_mode = 993 TOKEN_STRING_INITIALIZER(struct cmd_set_fwd_mode_result, mode, 994 "file-trans#iofwd#txonly#rxonly"); 995 996 cmdline_parse_inst_t cmd_set_fwd_mode = { 997 .f = cmd_set_fwd_mode_parsed, 998 .data = NULL, 999 .help_str = "set forwarding mode as file-trans|rxonly|txonly|iofwd", 1000 .tokens = { 1001 (void *)&cmd_setfwd_set, 1002 (void *)&cmd_setfwd_fwd, 1003 (void *)&cmd_setfwd_mode, 1004 NULL, 1005 }, 1006 }; 1007 1008 /* list of instructions */ 1009 cmdline_parse_ctx_t main_ctx[] = { 1010 (cmdline_parse_inst_t *)&cmd_help, 1011 (cmdline_parse_inst_t *)&cmd_send_file, 1012 (cmdline_parse_inst_t *)&cmd_start, 1013 (cmdline_parse_inst_t *)&cmd_stop, 1014 (cmdline_parse_inst_t *)&cmd_stats, 1015 (cmdline_parse_inst_t *)&cmd_set_fwd_mode, 1016 (cmdline_parse_inst_t *)&cmd_quit, 1017 NULL, 1018 }; 1019 1020 /* prompt function, called from main on MASTER lcore */ 1021 static void 1022 prompt(void) 1023 { 1024 struct cmdline *cl; 1025 1026 cl = cmdline_stdin_new(main_ctx, "ntb> "); 1027 if (cl == NULL) 1028 return; 1029 1030 cmdline_interact(cl); 1031 cmdline_stdin_exit(cl); 1032 } 1033 1034 static void 1035 signal_handler(int signum) 1036 { 1037 if (signum == SIGINT || signum == SIGTERM) { 1038 printf("\nSignal %d received, preparing to exit...\n", signum); 1039 signal(signum, SIG_DFL); 1040 kill(getpid(), signum); 1041 } 1042 } 1043 1044 #define OPT_BUF_SIZE "buf-size" 1045 #define OPT_FWD_MODE "fwd-mode" 1046 #define OPT_NB_DESC "nb-desc" 1047 #define OPT_TXFREET "txfreet" 1048 #define OPT_BURST "burst" 1049 #define OPT_QP "qp" 1050 1051 enum { 1052 /* long options mapped to a short option */ 1053 OPT_NO_ZERO_COPY_NUM = 1, 1054 OPT_BUF_SIZE_NUM, 1055 OPT_FWD_MODE_NUM, 1056 OPT_NB_DESC_NUM, 1057 OPT_TXFREET_NUM, 1058 OPT_BURST_NUM, 1059 OPT_QP_NUM, 1060 }; 1061 1062 static const char short_options[] = 1063 "i" /* interactive mode */ 1064 ; 1065 1066 static const struct option lgopts[] = { 1067 {OPT_BUF_SIZE, 1, NULL, OPT_BUF_SIZE_NUM }, 1068 {OPT_FWD_MODE, 1, NULL, OPT_FWD_MODE_NUM }, 1069 {OPT_NB_DESC, 1, NULL, OPT_NB_DESC_NUM }, 1070 {OPT_TXFREET, 1, NULL, OPT_TXFREET_NUM }, 1071 {OPT_BURST, 1, NULL, OPT_BURST_NUM }, 1072 {OPT_QP, 1, NULL, OPT_QP_NUM }, 1073 {0, 0, NULL, 0 } 1074 }; 1075 1076 static void 1077 ntb_usage(const char *prgname) 1078 { 1079 printf("%s [EAL options] -- [options]\n" 1080 "-i: run in interactive mode.\n" 1081 "-qp=N: set number of queues as N (N > 0, default: 1).\n" 1082 "--fwd-mode=N: set fwd mode (N: file-trans | rxonly | " 1083 "txonly | iofwd, default: file-trans)\n" 1084 "--buf-size=N: set mbuf dataroom size as N (0 < N < 65535," 1085 " default: 2048).\n" 1086 "--nb-desc=N: set number of descriptors as N (%u <= N <= %u," 1087 " default: 1024).\n" 1088 "--txfreet=N: set tx free thresh for NTB driver as N. (N >= 0)\n" 1089 "--burst=N: set pkt burst as N (0 < N <= %u default: 32).\n", 1090 prgname, NTB_MIN_DESC_SIZE, NTB_MAX_DESC_SIZE, 1091 NTB_MAX_PKT_BURST); 1092 } 1093 1094 static void 1095 ntb_parse_args(int argc, char **argv) 1096 { 1097 char *prgname = argv[0], **argvopt = argv; 1098 int opt, opt_idx, n, i; 1099 1100 while ((opt = getopt_long(argc, argvopt, short_options, 1101 lgopts, &opt_idx)) != EOF) { 1102 switch (opt) { 1103 case 'i': 1104 printf("Interactive-mode selected.\n"); 1105 interactive = 1; 1106 break; 1107 case OPT_QP_NUM: 1108 n = atoi(optarg); 1109 if (n > 0) 1110 num_queues = n; 1111 else 1112 rte_exit(EXIT_FAILURE, "q must be > 0.\n"); 1113 break; 1114 case OPT_BUF_SIZE_NUM: 1115 n = atoi(optarg); 1116 if (n > RTE_PKTMBUF_HEADROOM && n <= 0xFFFF) 1117 ntb_buf_size = n; 1118 else 1119 rte_exit(EXIT_FAILURE, "buf-size must be > " 1120 "%u and < 65536.\n", 1121 RTE_PKTMBUF_HEADROOM); 1122 break; 1123 case OPT_FWD_MODE_NUM: 1124 for (i = 0; i < MAX_FWD_MODE; i++) { 1125 if (!strcmp(optarg, fwd_mode_s[i])) { 1126 fwd_mode = i; 1127 break; 1128 } 1129 } 1130 if (i == MAX_FWD_MODE) 1131 rte_exit(EXIT_FAILURE, "Unsupported mode. " 1132 "(Should be: file-trans | rxonly | txonly " 1133 "| iofwd)\n"); 1134 break; 1135 case OPT_NB_DESC_NUM: 1136 n = atoi(optarg); 1137 if (n >= NTB_MIN_DESC_SIZE && n <= NTB_MAX_DESC_SIZE) 1138 nb_desc = n; 1139 else 1140 rte_exit(EXIT_FAILURE, "nb-desc must be within" 1141 " [%u, %u].\n", NTB_MIN_DESC_SIZE, 1142 NTB_MAX_DESC_SIZE); 1143 break; 1144 case OPT_TXFREET_NUM: 1145 n = atoi(optarg); 1146 if (n >= 0) 1147 tx_free_thresh = n; 1148 else 1149 rte_exit(EXIT_FAILURE, "txfreet must be" 1150 " >= 0\n"); 1151 break; 1152 case OPT_BURST_NUM: 1153 n = atoi(optarg); 1154 if (n > 0 && n <= NTB_MAX_PKT_BURST) 1155 pkt_burst = n; 1156 else 1157 rte_exit(EXIT_FAILURE, "burst must be within " 1158 "(0, %u].\n", NTB_MAX_PKT_BURST); 1159 break; 1160 1161 default: 1162 ntb_usage(prgname); 1163 rte_exit(EXIT_FAILURE, 1164 "Command line is incomplete or incorrect.\n"); 1165 break; 1166 } 1167 } 1168 } 1169 1170 static void 1171 ntb_mempool_mz_free(__rte_unused struct rte_mempool_memhdr *memhdr, 1172 void *opaque) 1173 { 1174 const struct rte_memzone *mz = opaque; 1175 rte_memzone_free(mz); 1176 } 1177 1178 static struct rte_mempool * 1179 ntb_mbuf_pool_create(uint16_t mbuf_seg_size, uint32_t nb_mbuf, 1180 struct ntb_dev_info ntb_info, 1181 struct ntb_dev_config *ntb_conf, 1182 unsigned int socket_id) 1183 { 1184 size_t mz_len, total_elt_sz, max_mz_len, left_sz; 1185 struct rte_pktmbuf_pool_private mbp_priv; 1186 char pool_name[RTE_MEMPOOL_NAMESIZE]; 1187 char mz_name[RTE_MEMZONE_NAMESIZE]; 1188 const struct rte_memzone *mz; 1189 struct rte_mempool *mp; 1190 uint64_t align; 1191 uint32_t mz_id; 1192 int ret; 1193 1194 snprintf(pool_name, sizeof(pool_name), "ntb_mbuf_pool_%u", socket_id); 1195 mp = rte_mempool_create_empty(pool_name, nb_mbuf, 1196 (mbuf_seg_size + sizeof(struct rte_mbuf)), 1197 MEMPOOL_CACHE_SIZE, 1198 sizeof(struct rte_pktmbuf_pool_private), 1199 socket_id, 0); 1200 if (mp == NULL) 1201 return NULL; 1202 1203 mbp_priv.mbuf_data_room_size = mbuf_seg_size; 1204 mbp_priv.mbuf_priv_size = 0; 1205 rte_pktmbuf_pool_init(mp, &mbp_priv); 1206 1207 ntb_conf->mz_list = rte_zmalloc("ntb_memzone_list", 1208 sizeof(struct rte_memzone *) * 1209 ntb_info.mw_cnt, 0); 1210 if (ntb_conf->mz_list == NULL) 1211 goto fail; 1212 1213 /* Put ntb header on mw0. */ 1214 if (ntb_info.mw_size[0] < ntb_info.ntb_hdr_size) { 1215 printf("mw0 (size: %" PRIu64 ") is not enough for ntb hdr" 1216 " (size: %u)\n", ntb_info.mw_size[0], 1217 ntb_info.ntb_hdr_size); 1218 goto fail; 1219 } 1220 1221 total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size; 1222 left_sz = total_elt_sz * nb_mbuf; 1223 for (mz_id = 0; mz_id < ntb_info.mw_cnt; mz_id++) { 1224 /* If populated mbuf is enough, no need to reserve extra mz. */ 1225 if (!left_sz) 1226 break; 1227 snprintf(mz_name, sizeof(mz_name), "ntb_mw_%d", mz_id); 1228 align = ntb_info.mw_size_align ? ntb_info.mw_size[mz_id] : 1229 RTE_CACHE_LINE_SIZE; 1230 /* Reserve ntb header space on memzone 0. */ 1231 max_mz_len = mz_id ? ntb_info.mw_size[mz_id] : 1232 ntb_info.mw_size[mz_id] - ntb_info.ntb_hdr_size; 1233 mz_len = left_sz <= max_mz_len ? left_sz : 1234 (max_mz_len / total_elt_sz * total_elt_sz); 1235 if (!mz_len) 1236 continue; 1237 mz = rte_memzone_reserve_aligned(mz_name, mz_len, socket_id, 1238 RTE_MEMZONE_IOVA_CONTIG, align); 1239 if (mz == NULL) { 1240 printf("Cannot allocate %" PRIu64 " aligned memzone" 1241 " %u\n", align, mz_id); 1242 goto fail; 1243 } 1244 left_sz -= mz_len; 1245 1246 /* Reserve ntb header space on memzone 0. */ 1247 if (mz_id) 1248 ret = rte_mempool_populate_iova(mp, mz->addr, mz->iova, 1249 mz->len, ntb_mempool_mz_free, 1250 (void *)(uintptr_t)mz); 1251 else 1252 ret = rte_mempool_populate_iova(mp, 1253 (void *)((size_t)mz->addr + 1254 ntb_info.ntb_hdr_size), 1255 mz->iova + ntb_info.ntb_hdr_size, 1256 mz->len - ntb_info.ntb_hdr_size, 1257 ntb_mempool_mz_free, 1258 (void *)(uintptr_t)mz); 1259 if (ret < 0) { 1260 rte_memzone_free(mz); 1261 rte_mempool_free(mp); 1262 return NULL; 1263 } 1264 1265 ntb_conf->mz_list[mz_id] = mz; 1266 } 1267 if (left_sz) { 1268 printf("mw space is not enough for mempool.\n"); 1269 goto fail; 1270 } 1271 1272 ntb_conf->mz_num = mz_id; 1273 rte_mempool_obj_iter(mp, rte_pktmbuf_init, NULL); 1274 1275 return mp; 1276 fail: 1277 rte_mempool_free(mp); 1278 return NULL; 1279 } 1280 1281 int 1282 main(int argc, char **argv) 1283 { 1284 struct rte_eth_conf eth_pconf = eth_port_conf; 1285 struct rte_rawdev_info ntb_rawdev_conf; 1286 struct rte_rawdev_info ntb_rawdev_info; 1287 struct rte_eth_dev_info ethdev_info; 1288 struct rte_eth_rxconf eth_rx_conf; 1289 struct rte_eth_txconf eth_tx_conf; 1290 struct ntb_queue_conf ntb_q_conf; 1291 struct ntb_dev_config ntb_conf; 1292 struct ntb_dev_info ntb_info; 1293 uint64_t ntb_link_status; 1294 uint32_t nb_mbuf; 1295 int ret, i; 1296 1297 signal(SIGINT, signal_handler); 1298 signal(SIGTERM, signal_handler); 1299 1300 ret = rte_eal_init(argc, argv); 1301 if (ret < 0) 1302 rte_exit(EXIT_FAILURE, "Error with EAL initialization.\n"); 1303 1304 if (rte_lcore_count() < 2) 1305 rte_exit(EXIT_FAILURE, "Need at least 2 cores\n"); 1306 1307 /* Find 1st ntb rawdev. */ 1308 for (i = 0; i < RTE_RAWDEV_MAX_DEVS; i++) 1309 if (rte_rawdevs[i].driver_name && 1310 (strncmp(rte_rawdevs[i].driver_name, "raw_ntb", 1311 NTB_DRV_NAME_LEN) == 0) && (rte_rawdevs[i].attached == 1)) 1312 break; 1313 1314 if (i == RTE_RAWDEV_MAX_DEVS) 1315 rte_exit(EXIT_FAILURE, "Cannot find any ntb device.\n"); 1316 1317 dev_id = i; 1318 1319 argc -= ret; 1320 argv += ret; 1321 1322 ntb_parse_args(argc, argv); 1323 1324 rte_rawdev_set_attr(dev_id, NTB_QUEUE_SZ_NAME, nb_desc); 1325 printf("Set queue size as %u.\n", nb_desc); 1326 rte_rawdev_set_attr(dev_id, NTB_QUEUE_NUM_NAME, num_queues); 1327 printf("Set queue number as %u.\n", num_queues); 1328 ntb_rawdev_info.dev_private = (rte_rawdev_obj_t)(&ntb_info); 1329 rte_rawdev_info_get(dev_id, &ntb_rawdev_info); 1330 1331 nb_mbuf = nb_desc * num_queues * 2 * 2 + rte_lcore_count() * 1332 MEMPOOL_CACHE_SIZE; 1333 mbuf_pool = ntb_mbuf_pool_create(ntb_buf_size, nb_mbuf, ntb_info, 1334 &ntb_conf, rte_socket_id()); 1335 if (mbuf_pool == NULL) 1336 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool.\n"); 1337 1338 ntb_conf.num_queues = num_queues; 1339 ntb_conf.queue_size = nb_desc; 1340 ntb_rawdev_conf.dev_private = (rte_rawdev_obj_t)(&ntb_conf); 1341 ret = rte_rawdev_configure(dev_id, &ntb_rawdev_conf); 1342 if (ret) 1343 rte_exit(EXIT_FAILURE, "Can't config ntb dev: err=%d, " 1344 "port=%u\n", ret, dev_id); 1345 1346 ntb_q_conf.tx_free_thresh = tx_free_thresh; 1347 ntb_q_conf.nb_desc = nb_desc; 1348 ntb_q_conf.rx_mp = mbuf_pool; 1349 for (i = 0; i < num_queues; i++) { 1350 /* Setup rawdev queue */ 1351 ret = rte_rawdev_queue_setup(dev_id, i, &ntb_q_conf); 1352 if (ret < 0) 1353 rte_exit(EXIT_FAILURE, 1354 "Failed to setup ntb queue %u.\n", i); 1355 } 1356 1357 /* Waiting for peer dev up at most 100s.*/ 1358 printf("Checking ntb link status...\n"); 1359 for (i = 0; i < 1000; i++) { 1360 rte_rawdev_get_attr(dev_id, NTB_LINK_STATUS_NAME, 1361 &ntb_link_status); 1362 if (ntb_link_status) { 1363 printf("Peer dev ready, ntb link up.\n"); 1364 break; 1365 } 1366 rte_delay_ms(100); 1367 } 1368 rte_rawdev_get_attr(dev_id, NTB_LINK_STATUS_NAME, &ntb_link_status); 1369 if (ntb_link_status == 0) 1370 printf("Expire 100s. Link is not up. Please restart app.\n"); 1371 1372 ret = rte_rawdev_start(dev_id); 1373 if (ret < 0) 1374 rte_exit(EXIT_FAILURE, "rte_rawdev_start: err=%d, port=%u\n", 1375 ret, dev_id); 1376 1377 /* Find 1st ethdev */ 1378 eth_port_id = rte_eth_find_next(0); 1379 1380 if (eth_port_id < RTE_MAX_ETHPORTS) { 1381 rte_eth_dev_info_get(eth_port_id, ðdev_info); 1382 eth_pconf.rx_adv_conf.rss_conf.rss_hf &= 1383 ethdev_info.flow_type_rss_offloads; 1384 ret = rte_eth_dev_configure(eth_port_id, num_queues, 1385 num_queues, ð_pconf); 1386 if (ret) 1387 rte_exit(EXIT_FAILURE, "Can't config ethdev: err=%d, " 1388 "port=%u\n", ret, eth_port_id); 1389 eth_rx_conf = ethdev_info.default_rxconf; 1390 eth_rx_conf.offloads = eth_pconf.rxmode.offloads; 1391 eth_tx_conf = ethdev_info.default_txconf; 1392 eth_tx_conf.offloads = eth_pconf.txmode.offloads; 1393 1394 /* Setup ethdev queue if ethdev exists */ 1395 for (i = 0; i < num_queues; i++) { 1396 ret = rte_eth_rx_queue_setup(eth_port_id, i, nb_desc, 1397 rte_eth_dev_socket_id(eth_port_id), 1398 ð_rx_conf, mbuf_pool); 1399 if (ret < 0) 1400 rte_exit(EXIT_FAILURE, 1401 "Failed to setup eth rxq %u.\n", i); 1402 ret = rte_eth_tx_queue_setup(eth_port_id, i, nb_desc, 1403 rte_eth_dev_socket_id(eth_port_id), 1404 ð_tx_conf); 1405 if (ret < 0) 1406 rte_exit(EXIT_FAILURE, 1407 "Failed to setup eth txq %u.\n", i); 1408 } 1409 1410 ret = rte_eth_dev_start(eth_port_id); 1411 if (ret < 0) 1412 rte_exit(EXIT_FAILURE, "rte_eth_dev_start: err=%d, " 1413 "port=%u\n", ret, eth_port_id); 1414 } 1415 1416 /* initialize port stats */ 1417 memset(&ntb_port_stats, 0, sizeof(ntb_port_stats)); 1418 1419 /* Set default fwd mode if user doesn't set it. */ 1420 if (fwd_mode == MAX_FWD_MODE && eth_port_id < RTE_MAX_ETHPORTS) { 1421 printf("Set default fwd mode as iofwd.\n"); 1422 fwd_mode = IOFWD; 1423 } 1424 if (fwd_mode == MAX_FWD_MODE) { 1425 printf("Set default fwd mode as file-trans.\n"); 1426 fwd_mode = FILE_TRANS; 1427 } 1428 1429 if (interactive) { 1430 sleep(1); 1431 prompt(); 1432 } else { 1433 start_pkt_fwd(); 1434 } 1435 1436 return 0; 1437 } 1438