1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Cavium, Inc 3 */ 4 5 #include <rte_string_fns.h> 6 #include <rte_ethdev_driver.h> 7 #include <rte_ethdev_pci.h> 8 #include <rte_cycles.h> 9 #include <rte_malloc.h> 10 #include <rte_alarm.h> 11 #include <rte_ether.h> 12 13 #include "lio_logs.h" 14 #include "lio_23xx_vf.h" 15 #include "lio_ethdev.h" 16 #include "lio_rxtx.h" 17 18 int lio_logtype_init; 19 int lio_logtype_driver; 20 21 /* Default RSS key in use */ 22 static uint8_t lio_rss_key[40] = { 23 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2, 24 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0, 25 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4, 26 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C, 27 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA, 28 }; 29 30 static const struct rte_eth_desc_lim lio_rx_desc_lim = { 31 .nb_max = CN23XX_MAX_OQ_DESCRIPTORS, 32 .nb_min = CN23XX_MIN_OQ_DESCRIPTORS, 33 .nb_align = 1, 34 }; 35 36 static const struct rte_eth_desc_lim lio_tx_desc_lim = { 37 .nb_max = CN23XX_MAX_IQ_DESCRIPTORS, 38 .nb_min = CN23XX_MIN_IQ_DESCRIPTORS, 39 .nb_align = 1, 40 }; 41 42 /* Wait for control command to reach nic. */ 43 static uint16_t 44 lio_wait_for_ctrl_cmd(struct lio_device *lio_dev, 45 struct lio_dev_ctrl_cmd *ctrl_cmd) 46 { 47 uint16_t timeout = LIO_MAX_CMD_TIMEOUT; 48 49 while ((ctrl_cmd->cond == 0) && --timeout) { 50 lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); 51 rte_delay_ms(1); 52 } 53 54 return !timeout; 55 } 56 57 /** 58 * \brief Send Rx control command 59 * @param eth_dev Pointer to the structure rte_eth_dev 60 * @param start_stop whether to start or stop 61 */ 62 static int 63 lio_send_rx_ctrl_cmd(struct rte_eth_dev *eth_dev, int start_stop) 64 { 65 struct lio_device *lio_dev = LIO_DEV(eth_dev); 66 struct lio_dev_ctrl_cmd ctrl_cmd; 67 struct lio_ctrl_pkt ctrl_pkt; 68 69 /* flush added to prevent cmd failure 70 * incase the queue is full 71 */ 72 lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); 73 74 memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); 75 memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); 76 77 ctrl_cmd.eth_dev = eth_dev; 78 ctrl_cmd.cond = 0; 79 80 ctrl_pkt.ncmd.s.cmd = LIO_CMD_RX_CTL; 81 ctrl_pkt.ncmd.s.param1 = start_stop; 82 ctrl_pkt.ctrl_cmd = &ctrl_cmd; 83 84 if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { 85 lio_dev_err(lio_dev, "Failed to send RX Control message\n"); 86 return -1; 87 } 88 89 if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) { 90 lio_dev_err(lio_dev, "RX Control command timed out\n"); 91 return -1; 92 } 93 94 return 0; 95 } 96 97 /* store statistics names and its offset in stats structure */ 98 struct rte_lio_xstats_name_off { 99 char name[RTE_ETH_XSTATS_NAME_SIZE]; 100 unsigned int offset; 101 }; 102 103 static const struct rte_lio_xstats_name_off rte_lio_stats_strings[] = { 104 {"rx_pkts", offsetof(struct octeon_rx_stats, total_rcvd)}, 105 {"rx_bytes", offsetof(struct octeon_rx_stats, bytes_rcvd)}, 106 {"rx_broadcast_pkts", offsetof(struct octeon_rx_stats, total_bcst)}, 107 {"rx_multicast_pkts", offsetof(struct octeon_rx_stats, total_mcst)}, 108 {"rx_flow_ctrl_pkts", offsetof(struct octeon_rx_stats, ctl_rcvd)}, 109 {"rx_fifo_err", offsetof(struct octeon_rx_stats, fifo_err)}, 110 {"rx_dmac_drop", offsetof(struct octeon_rx_stats, dmac_drop)}, 111 {"rx_fcs_err", offsetof(struct octeon_rx_stats, fcs_err)}, 112 {"rx_jabber_err", offsetof(struct octeon_rx_stats, jabber_err)}, 113 {"rx_l2_err", offsetof(struct octeon_rx_stats, l2_err)}, 114 {"rx_vxlan_pkts", offsetof(struct octeon_rx_stats, fw_rx_vxlan)}, 115 {"rx_vxlan_err", offsetof(struct octeon_rx_stats, fw_rx_vxlan_err)}, 116 {"rx_lro_pkts", offsetof(struct octeon_rx_stats, fw_lro_pkts)}, 117 {"tx_pkts", (offsetof(struct octeon_tx_stats, total_pkts_sent)) + 118 sizeof(struct octeon_rx_stats)}, 119 {"tx_bytes", (offsetof(struct octeon_tx_stats, total_bytes_sent)) + 120 sizeof(struct octeon_rx_stats)}, 121 {"tx_broadcast_pkts", 122 (offsetof(struct octeon_tx_stats, bcast_pkts_sent)) + 123 sizeof(struct octeon_rx_stats)}, 124 {"tx_multicast_pkts", 125 (offsetof(struct octeon_tx_stats, mcast_pkts_sent)) + 126 sizeof(struct octeon_rx_stats)}, 127 {"tx_flow_ctrl_pkts", (offsetof(struct octeon_tx_stats, ctl_sent)) + 128 sizeof(struct octeon_rx_stats)}, 129 {"tx_fifo_err", (offsetof(struct octeon_tx_stats, fifo_err)) + 130 sizeof(struct octeon_rx_stats)}, 131 {"tx_total_collisions", (offsetof(struct octeon_tx_stats, 132 total_collisions)) + 133 sizeof(struct octeon_rx_stats)}, 134 {"tx_tso", (offsetof(struct octeon_tx_stats, fw_tso)) + 135 sizeof(struct octeon_rx_stats)}, 136 {"tx_vxlan_pkts", (offsetof(struct octeon_tx_stats, fw_tx_vxlan)) + 137 sizeof(struct octeon_rx_stats)}, 138 }; 139 140 #define LIO_NB_XSTATS RTE_DIM(rte_lio_stats_strings) 141 142 /* Get hw stats of the port */ 143 static int 144 lio_dev_xstats_get(struct rte_eth_dev *eth_dev, struct rte_eth_xstat *xstats, 145 unsigned int n) 146 { 147 struct lio_device *lio_dev = LIO_DEV(eth_dev); 148 uint16_t timeout = LIO_MAX_CMD_TIMEOUT; 149 struct octeon_link_stats *hw_stats; 150 struct lio_link_stats_resp *resp; 151 struct lio_soft_command *sc; 152 uint32_t resp_size; 153 unsigned int i; 154 int retval; 155 156 if (!lio_dev->intf_open) { 157 lio_dev_err(lio_dev, "Port %d down\n", 158 lio_dev->port_id); 159 return -EINVAL; 160 } 161 162 if (n < LIO_NB_XSTATS) 163 return LIO_NB_XSTATS; 164 165 resp_size = sizeof(struct lio_link_stats_resp); 166 sc = lio_alloc_soft_command(lio_dev, 0, resp_size, 0); 167 if (sc == NULL) 168 return -ENOMEM; 169 170 resp = (struct lio_link_stats_resp *)sc->virtrptr; 171 lio_prepare_soft_command(lio_dev, sc, LIO_OPCODE, 172 LIO_OPCODE_PORT_STATS, 0, 0, 0); 173 174 /* Setting wait time in seconds */ 175 sc->wait_time = LIO_MAX_CMD_TIMEOUT / 1000; 176 177 retval = lio_send_soft_command(lio_dev, sc); 178 if (retval == LIO_IQ_SEND_FAILED) { 179 lio_dev_err(lio_dev, "failed to get port stats from firmware. status: %x\n", 180 retval); 181 goto get_stats_fail; 182 } 183 184 while ((*sc->status_word == LIO_COMPLETION_WORD_INIT) && --timeout) { 185 lio_flush_iq(lio_dev, lio_dev->instr_queue[sc->iq_no]); 186 lio_process_ordered_list(lio_dev); 187 rte_delay_ms(1); 188 } 189 190 retval = resp->status; 191 if (retval) { 192 lio_dev_err(lio_dev, "failed to get port stats from firmware\n"); 193 goto get_stats_fail; 194 } 195 196 lio_swap_8B_data((uint64_t *)(&resp->link_stats), 197 sizeof(struct octeon_link_stats) >> 3); 198 199 hw_stats = &resp->link_stats; 200 201 for (i = 0; i < LIO_NB_XSTATS; i++) { 202 xstats[i].id = i; 203 xstats[i].value = 204 *(uint64_t *)(((char *)hw_stats) + 205 rte_lio_stats_strings[i].offset); 206 } 207 208 lio_free_soft_command(sc); 209 210 return LIO_NB_XSTATS; 211 212 get_stats_fail: 213 lio_free_soft_command(sc); 214 215 return -1; 216 } 217 218 static int 219 lio_dev_xstats_get_names(struct rte_eth_dev *eth_dev, 220 struct rte_eth_xstat_name *xstats_names, 221 unsigned limit __rte_unused) 222 { 223 struct lio_device *lio_dev = LIO_DEV(eth_dev); 224 unsigned int i; 225 226 if (!lio_dev->intf_open) { 227 lio_dev_err(lio_dev, "Port %d down\n", 228 lio_dev->port_id); 229 return -EINVAL; 230 } 231 232 if (xstats_names == NULL) 233 return LIO_NB_XSTATS; 234 235 /* Note: limit checked in rte_eth_xstats_names() */ 236 237 for (i = 0; i < LIO_NB_XSTATS; i++) { 238 snprintf(xstats_names[i].name, sizeof(xstats_names[i].name), 239 "%s", rte_lio_stats_strings[i].name); 240 } 241 242 return LIO_NB_XSTATS; 243 } 244 245 /* Reset hw stats for the port */ 246 static void 247 lio_dev_xstats_reset(struct rte_eth_dev *eth_dev) 248 { 249 struct lio_device *lio_dev = LIO_DEV(eth_dev); 250 struct lio_dev_ctrl_cmd ctrl_cmd; 251 struct lio_ctrl_pkt ctrl_pkt; 252 253 if (!lio_dev->intf_open) { 254 lio_dev_err(lio_dev, "Port %d down\n", 255 lio_dev->port_id); 256 return; 257 } 258 259 /* flush added to prevent cmd failure 260 * incase the queue is full 261 */ 262 lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); 263 264 memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); 265 memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); 266 267 ctrl_cmd.eth_dev = eth_dev; 268 ctrl_cmd.cond = 0; 269 270 ctrl_pkt.ncmd.s.cmd = LIO_CMD_CLEAR_STATS; 271 ctrl_pkt.ctrl_cmd = &ctrl_cmd; 272 273 if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { 274 lio_dev_err(lio_dev, "Failed to send clear stats command\n"); 275 return; 276 } 277 278 if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) { 279 lio_dev_err(lio_dev, "Clear stats command timed out\n"); 280 return; 281 } 282 283 /* clear stored per queue stats */ 284 RTE_FUNC_PTR_OR_RET(*eth_dev->dev_ops->stats_reset); 285 (*eth_dev->dev_ops->stats_reset)(eth_dev); 286 } 287 288 /* Retrieve the device statistics (# packets in/out, # bytes in/out, etc */ 289 static int 290 lio_dev_stats_get(struct rte_eth_dev *eth_dev, 291 struct rte_eth_stats *stats) 292 { 293 struct lio_device *lio_dev = LIO_DEV(eth_dev); 294 struct lio_droq_stats *oq_stats; 295 struct lio_iq_stats *iq_stats; 296 struct lio_instr_queue *txq; 297 struct lio_droq *droq; 298 int i, iq_no, oq_no; 299 uint64_t bytes = 0; 300 uint64_t pkts = 0; 301 uint64_t drop = 0; 302 303 for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { 304 iq_no = lio_dev->linfo.txpciq[i].s.q_no; 305 txq = lio_dev->instr_queue[iq_no]; 306 if (txq != NULL) { 307 iq_stats = &txq->stats; 308 pkts += iq_stats->tx_done; 309 drop += iq_stats->tx_dropped; 310 bytes += iq_stats->tx_tot_bytes; 311 } 312 } 313 314 stats->opackets = pkts; 315 stats->obytes = bytes; 316 stats->oerrors = drop; 317 318 pkts = 0; 319 drop = 0; 320 bytes = 0; 321 322 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { 323 oq_no = lio_dev->linfo.rxpciq[i].s.q_no; 324 droq = lio_dev->droq[oq_no]; 325 if (droq != NULL) { 326 oq_stats = &droq->stats; 327 pkts += oq_stats->rx_pkts_received; 328 drop += (oq_stats->rx_dropped + 329 oq_stats->dropped_toomany + 330 oq_stats->dropped_nomem); 331 bytes += oq_stats->rx_bytes_received; 332 } 333 } 334 stats->ibytes = bytes; 335 stats->ipackets = pkts; 336 stats->ierrors = drop; 337 338 return 0; 339 } 340 341 static void 342 lio_dev_stats_reset(struct rte_eth_dev *eth_dev) 343 { 344 struct lio_device *lio_dev = LIO_DEV(eth_dev); 345 struct lio_droq_stats *oq_stats; 346 struct lio_iq_stats *iq_stats; 347 struct lio_instr_queue *txq; 348 struct lio_droq *droq; 349 int i, iq_no, oq_no; 350 351 for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { 352 iq_no = lio_dev->linfo.txpciq[i].s.q_no; 353 txq = lio_dev->instr_queue[iq_no]; 354 if (txq != NULL) { 355 iq_stats = &txq->stats; 356 memset(iq_stats, 0, sizeof(struct lio_iq_stats)); 357 } 358 } 359 360 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { 361 oq_no = lio_dev->linfo.rxpciq[i].s.q_no; 362 droq = lio_dev->droq[oq_no]; 363 if (droq != NULL) { 364 oq_stats = &droq->stats; 365 memset(oq_stats, 0, sizeof(struct lio_droq_stats)); 366 } 367 } 368 } 369 370 static void 371 lio_dev_info_get(struct rte_eth_dev *eth_dev, 372 struct rte_eth_dev_info *devinfo) 373 { 374 struct lio_device *lio_dev = LIO_DEV(eth_dev); 375 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 376 377 switch (pci_dev->id.subsystem_device_id) { 378 /* CN23xx 10G cards */ 379 case PCI_SUBSYS_DEV_ID_CN2350_210: 380 case PCI_SUBSYS_DEV_ID_CN2360_210: 381 case PCI_SUBSYS_DEV_ID_CN2350_210SVPN3: 382 case PCI_SUBSYS_DEV_ID_CN2360_210SVPN3: 383 case PCI_SUBSYS_DEV_ID_CN2350_210SVPT: 384 case PCI_SUBSYS_DEV_ID_CN2360_210SVPT: 385 devinfo->speed_capa = ETH_LINK_SPEED_10G; 386 break; 387 /* CN23xx 25G cards */ 388 case PCI_SUBSYS_DEV_ID_CN2350_225: 389 case PCI_SUBSYS_DEV_ID_CN2360_225: 390 devinfo->speed_capa = ETH_LINK_SPEED_25G; 391 break; 392 default: 393 devinfo->speed_capa = ETH_LINK_SPEED_10G; 394 lio_dev_err(lio_dev, 395 "Unknown CN23XX subsystem device id. Setting 10G as default link speed.\n"); 396 } 397 398 devinfo->max_rx_queues = lio_dev->max_rx_queues; 399 devinfo->max_tx_queues = lio_dev->max_tx_queues; 400 401 devinfo->min_rx_bufsize = LIO_MIN_RX_BUF_SIZE; 402 devinfo->max_rx_pktlen = LIO_MAX_RX_PKTLEN; 403 404 devinfo->max_mac_addrs = 1; 405 406 devinfo->rx_offload_capa = (DEV_RX_OFFLOAD_IPV4_CKSUM | 407 DEV_RX_OFFLOAD_UDP_CKSUM | 408 DEV_RX_OFFLOAD_TCP_CKSUM | 409 DEV_RX_OFFLOAD_VLAN_STRIP); 410 devinfo->tx_offload_capa = (DEV_TX_OFFLOAD_IPV4_CKSUM | 411 DEV_TX_OFFLOAD_UDP_CKSUM | 412 DEV_TX_OFFLOAD_TCP_CKSUM | 413 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM); 414 415 devinfo->rx_desc_lim = lio_rx_desc_lim; 416 devinfo->tx_desc_lim = lio_tx_desc_lim; 417 418 devinfo->reta_size = LIO_RSS_MAX_TABLE_SZ; 419 devinfo->hash_key_size = LIO_RSS_MAX_KEY_SZ; 420 devinfo->flow_type_rss_offloads = (ETH_RSS_IPV4 | 421 ETH_RSS_NONFRAG_IPV4_TCP | 422 ETH_RSS_IPV6 | 423 ETH_RSS_NONFRAG_IPV6_TCP | 424 ETH_RSS_IPV6_EX | 425 ETH_RSS_IPV6_TCP_EX); 426 } 427 428 static int 429 lio_dev_mtu_set(struct rte_eth_dev *eth_dev, uint16_t mtu) 430 { 431 struct lio_device *lio_dev = LIO_DEV(eth_dev); 432 uint16_t pf_mtu = lio_dev->linfo.link.s.mtu; 433 uint32_t frame_len = mtu + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 434 struct lio_dev_ctrl_cmd ctrl_cmd; 435 struct lio_ctrl_pkt ctrl_pkt; 436 437 PMD_INIT_FUNC_TRACE(); 438 439 if (!lio_dev->intf_open) { 440 lio_dev_err(lio_dev, "Port %d down, can't set MTU\n", 441 lio_dev->port_id); 442 return -EINVAL; 443 } 444 445 /* check if VF MTU is within allowed range. 446 * New value should not exceed PF MTU. 447 */ 448 if (mtu < RTE_ETHER_MIN_MTU || mtu > pf_mtu) { 449 lio_dev_err(lio_dev, "VF MTU should be >= %d and <= %d\n", 450 RTE_ETHER_MIN_MTU, pf_mtu); 451 return -EINVAL; 452 } 453 454 /* flush added to prevent cmd failure 455 * incase the queue is full 456 */ 457 lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); 458 459 memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); 460 memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); 461 462 ctrl_cmd.eth_dev = eth_dev; 463 ctrl_cmd.cond = 0; 464 465 ctrl_pkt.ncmd.s.cmd = LIO_CMD_CHANGE_MTU; 466 ctrl_pkt.ncmd.s.param1 = mtu; 467 ctrl_pkt.ctrl_cmd = &ctrl_cmd; 468 469 if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { 470 lio_dev_err(lio_dev, "Failed to send command to change MTU\n"); 471 return -1; 472 } 473 474 if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) { 475 lio_dev_err(lio_dev, "Command to change MTU timed out\n"); 476 return -1; 477 } 478 479 if (frame_len > RTE_ETHER_MAX_LEN) 480 eth_dev->data->dev_conf.rxmode.offloads |= 481 DEV_RX_OFFLOAD_JUMBO_FRAME; 482 else 483 eth_dev->data->dev_conf.rxmode.offloads &= 484 ~DEV_RX_OFFLOAD_JUMBO_FRAME; 485 486 eth_dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_len; 487 eth_dev->data->mtu = mtu; 488 489 return 0; 490 } 491 492 static int 493 lio_dev_rss_reta_update(struct rte_eth_dev *eth_dev, 494 struct rte_eth_rss_reta_entry64 *reta_conf, 495 uint16_t reta_size) 496 { 497 struct lio_device *lio_dev = LIO_DEV(eth_dev); 498 struct lio_rss_ctx *rss_state = &lio_dev->rss_state; 499 struct lio_rss_set *rss_param; 500 struct lio_dev_ctrl_cmd ctrl_cmd; 501 struct lio_ctrl_pkt ctrl_pkt; 502 int i, j, index; 503 504 if (!lio_dev->intf_open) { 505 lio_dev_err(lio_dev, "Port %d down, can't update reta\n", 506 lio_dev->port_id); 507 return -EINVAL; 508 } 509 510 if (reta_size != LIO_RSS_MAX_TABLE_SZ) { 511 lio_dev_err(lio_dev, 512 "The size of hash lookup table configured (%d) doesn't match the number hardware can supported (%d)\n", 513 reta_size, LIO_RSS_MAX_TABLE_SZ); 514 return -EINVAL; 515 } 516 517 /* flush added to prevent cmd failure 518 * incase the queue is full 519 */ 520 lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); 521 522 memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); 523 memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); 524 525 rss_param = (struct lio_rss_set *)&ctrl_pkt.udd[0]; 526 527 ctrl_cmd.eth_dev = eth_dev; 528 ctrl_cmd.cond = 0; 529 530 ctrl_pkt.ncmd.s.cmd = LIO_CMD_SET_RSS; 531 ctrl_pkt.ncmd.s.more = sizeof(struct lio_rss_set) >> 3; 532 ctrl_pkt.ctrl_cmd = &ctrl_cmd; 533 534 rss_param->param.flags = 0xF; 535 rss_param->param.flags &= ~LIO_RSS_PARAM_ITABLE_UNCHANGED; 536 rss_param->param.itablesize = LIO_RSS_MAX_TABLE_SZ; 537 538 for (i = 0; i < (reta_size / RTE_RETA_GROUP_SIZE); i++) { 539 for (j = 0; j < RTE_RETA_GROUP_SIZE; j++) { 540 if ((reta_conf[i].mask) & ((uint64_t)1 << j)) { 541 index = (i * RTE_RETA_GROUP_SIZE) + j; 542 rss_state->itable[index] = reta_conf[i].reta[j]; 543 } 544 } 545 } 546 547 rss_state->itable_size = LIO_RSS_MAX_TABLE_SZ; 548 memcpy(rss_param->itable, rss_state->itable, rss_state->itable_size); 549 550 lio_swap_8B_data((uint64_t *)rss_param, LIO_RSS_PARAM_SIZE >> 3); 551 552 if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { 553 lio_dev_err(lio_dev, "Failed to set rss hash\n"); 554 return -1; 555 } 556 557 if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) { 558 lio_dev_err(lio_dev, "Set rss hash timed out\n"); 559 return -1; 560 } 561 562 return 0; 563 } 564 565 static int 566 lio_dev_rss_reta_query(struct rte_eth_dev *eth_dev, 567 struct rte_eth_rss_reta_entry64 *reta_conf, 568 uint16_t reta_size) 569 { 570 struct lio_device *lio_dev = LIO_DEV(eth_dev); 571 struct lio_rss_ctx *rss_state = &lio_dev->rss_state; 572 int i, num; 573 574 if (reta_size != LIO_RSS_MAX_TABLE_SZ) { 575 lio_dev_err(lio_dev, 576 "The size of hash lookup table configured (%d) doesn't match the number hardware can supported (%d)\n", 577 reta_size, LIO_RSS_MAX_TABLE_SZ); 578 return -EINVAL; 579 } 580 581 num = reta_size / RTE_RETA_GROUP_SIZE; 582 583 for (i = 0; i < num; i++) { 584 memcpy(reta_conf->reta, 585 &rss_state->itable[i * RTE_RETA_GROUP_SIZE], 586 RTE_RETA_GROUP_SIZE); 587 reta_conf++; 588 } 589 590 return 0; 591 } 592 593 static int 594 lio_dev_rss_hash_conf_get(struct rte_eth_dev *eth_dev, 595 struct rte_eth_rss_conf *rss_conf) 596 { 597 struct lio_device *lio_dev = LIO_DEV(eth_dev); 598 struct lio_rss_ctx *rss_state = &lio_dev->rss_state; 599 uint8_t *hash_key = NULL; 600 uint64_t rss_hf = 0; 601 602 if (rss_state->hash_disable) { 603 lio_dev_info(lio_dev, "RSS disabled in nic\n"); 604 rss_conf->rss_hf = 0; 605 return 0; 606 } 607 608 /* Get key value */ 609 hash_key = rss_conf->rss_key; 610 if (hash_key != NULL) 611 memcpy(hash_key, rss_state->hash_key, rss_state->hash_key_size); 612 613 if (rss_state->ip) 614 rss_hf |= ETH_RSS_IPV4; 615 if (rss_state->tcp_hash) 616 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP; 617 if (rss_state->ipv6) 618 rss_hf |= ETH_RSS_IPV6; 619 if (rss_state->ipv6_tcp_hash) 620 rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP; 621 if (rss_state->ipv6_ex) 622 rss_hf |= ETH_RSS_IPV6_EX; 623 if (rss_state->ipv6_tcp_ex_hash) 624 rss_hf |= ETH_RSS_IPV6_TCP_EX; 625 626 rss_conf->rss_hf = rss_hf; 627 628 return 0; 629 } 630 631 static int 632 lio_dev_rss_hash_update(struct rte_eth_dev *eth_dev, 633 struct rte_eth_rss_conf *rss_conf) 634 { 635 struct lio_device *lio_dev = LIO_DEV(eth_dev); 636 struct lio_rss_ctx *rss_state = &lio_dev->rss_state; 637 struct lio_rss_set *rss_param; 638 struct lio_dev_ctrl_cmd ctrl_cmd; 639 struct lio_ctrl_pkt ctrl_pkt; 640 641 if (!lio_dev->intf_open) { 642 lio_dev_err(lio_dev, "Port %d down, can't update hash\n", 643 lio_dev->port_id); 644 return -EINVAL; 645 } 646 647 /* flush added to prevent cmd failure 648 * incase the queue is full 649 */ 650 lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); 651 652 memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); 653 memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); 654 655 rss_param = (struct lio_rss_set *)&ctrl_pkt.udd[0]; 656 657 ctrl_cmd.eth_dev = eth_dev; 658 ctrl_cmd.cond = 0; 659 660 ctrl_pkt.ncmd.s.cmd = LIO_CMD_SET_RSS; 661 ctrl_pkt.ncmd.s.more = sizeof(struct lio_rss_set) >> 3; 662 ctrl_pkt.ctrl_cmd = &ctrl_cmd; 663 664 rss_param->param.flags = 0xF; 665 666 if (rss_conf->rss_key) { 667 rss_param->param.flags &= ~LIO_RSS_PARAM_HASH_KEY_UNCHANGED; 668 rss_state->hash_key_size = LIO_RSS_MAX_KEY_SZ; 669 rss_param->param.hashkeysize = LIO_RSS_MAX_KEY_SZ; 670 memcpy(rss_state->hash_key, rss_conf->rss_key, 671 rss_state->hash_key_size); 672 memcpy(rss_param->key, rss_state->hash_key, 673 rss_state->hash_key_size); 674 } 675 676 if ((rss_conf->rss_hf & LIO_RSS_OFFLOAD_ALL) == 0) { 677 /* Can't disable rss through hash flags, 678 * if it is enabled by default during init 679 */ 680 if (!rss_state->hash_disable) 681 return -EINVAL; 682 683 /* This is for --disable-rss during testpmd launch */ 684 rss_param->param.flags |= LIO_RSS_PARAM_DISABLE_RSS; 685 } else { 686 uint32_t hashinfo = 0; 687 688 /* Can't enable rss if disabled by default during init */ 689 if (rss_state->hash_disable) 690 return -EINVAL; 691 692 if (rss_conf->rss_hf & ETH_RSS_IPV4) { 693 hashinfo |= LIO_RSS_HASH_IPV4; 694 rss_state->ip = 1; 695 } else { 696 rss_state->ip = 0; 697 } 698 699 if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) { 700 hashinfo |= LIO_RSS_HASH_TCP_IPV4; 701 rss_state->tcp_hash = 1; 702 } else { 703 rss_state->tcp_hash = 0; 704 } 705 706 if (rss_conf->rss_hf & ETH_RSS_IPV6) { 707 hashinfo |= LIO_RSS_HASH_IPV6; 708 rss_state->ipv6 = 1; 709 } else { 710 rss_state->ipv6 = 0; 711 } 712 713 if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) { 714 hashinfo |= LIO_RSS_HASH_TCP_IPV6; 715 rss_state->ipv6_tcp_hash = 1; 716 } else { 717 rss_state->ipv6_tcp_hash = 0; 718 } 719 720 if (rss_conf->rss_hf & ETH_RSS_IPV6_EX) { 721 hashinfo |= LIO_RSS_HASH_IPV6_EX; 722 rss_state->ipv6_ex = 1; 723 } else { 724 rss_state->ipv6_ex = 0; 725 } 726 727 if (rss_conf->rss_hf & ETH_RSS_IPV6_TCP_EX) { 728 hashinfo |= LIO_RSS_HASH_TCP_IPV6_EX; 729 rss_state->ipv6_tcp_ex_hash = 1; 730 } else { 731 rss_state->ipv6_tcp_ex_hash = 0; 732 } 733 734 rss_param->param.flags &= ~LIO_RSS_PARAM_HASH_INFO_UNCHANGED; 735 rss_param->param.hashinfo = hashinfo; 736 } 737 738 lio_swap_8B_data((uint64_t *)rss_param, LIO_RSS_PARAM_SIZE >> 3); 739 740 if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { 741 lio_dev_err(lio_dev, "Failed to set rss hash\n"); 742 return -1; 743 } 744 745 if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) { 746 lio_dev_err(lio_dev, "Set rss hash timed out\n"); 747 return -1; 748 } 749 750 return 0; 751 } 752 753 /** 754 * Add vxlan dest udp port for an interface. 755 * 756 * @param eth_dev 757 * Pointer to the structure rte_eth_dev 758 * @param udp_tnl 759 * udp tunnel conf 760 * 761 * @return 762 * On success return 0 763 * On failure return -1 764 */ 765 static int 766 lio_dev_udp_tunnel_add(struct rte_eth_dev *eth_dev, 767 struct rte_eth_udp_tunnel *udp_tnl) 768 { 769 struct lio_device *lio_dev = LIO_DEV(eth_dev); 770 struct lio_dev_ctrl_cmd ctrl_cmd; 771 struct lio_ctrl_pkt ctrl_pkt; 772 773 if (udp_tnl == NULL) 774 return -EINVAL; 775 776 if (udp_tnl->prot_type != RTE_TUNNEL_TYPE_VXLAN) { 777 lio_dev_err(lio_dev, "Unsupported tunnel type\n"); 778 return -1; 779 } 780 781 /* flush added to prevent cmd failure 782 * incase the queue is full 783 */ 784 lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); 785 786 memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); 787 memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); 788 789 ctrl_cmd.eth_dev = eth_dev; 790 ctrl_cmd.cond = 0; 791 792 ctrl_pkt.ncmd.s.cmd = LIO_CMD_VXLAN_PORT_CONFIG; 793 ctrl_pkt.ncmd.s.param1 = udp_tnl->udp_port; 794 ctrl_pkt.ncmd.s.more = LIO_CMD_VXLAN_PORT_ADD; 795 ctrl_pkt.ctrl_cmd = &ctrl_cmd; 796 797 if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { 798 lio_dev_err(lio_dev, "Failed to send VXLAN_PORT_ADD command\n"); 799 return -1; 800 } 801 802 if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) { 803 lio_dev_err(lio_dev, "VXLAN_PORT_ADD command timed out\n"); 804 return -1; 805 } 806 807 return 0; 808 } 809 810 /** 811 * Remove vxlan dest udp port for an interface. 812 * 813 * @param eth_dev 814 * Pointer to the structure rte_eth_dev 815 * @param udp_tnl 816 * udp tunnel conf 817 * 818 * @return 819 * On success return 0 820 * On failure return -1 821 */ 822 static int 823 lio_dev_udp_tunnel_del(struct rte_eth_dev *eth_dev, 824 struct rte_eth_udp_tunnel *udp_tnl) 825 { 826 struct lio_device *lio_dev = LIO_DEV(eth_dev); 827 struct lio_dev_ctrl_cmd ctrl_cmd; 828 struct lio_ctrl_pkt ctrl_pkt; 829 830 if (udp_tnl == NULL) 831 return -EINVAL; 832 833 if (udp_tnl->prot_type != RTE_TUNNEL_TYPE_VXLAN) { 834 lio_dev_err(lio_dev, "Unsupported tunnel type\n"); 835 return -1; 836 } 837 838 /* flush added to prevent cmd failure 839 * incase the queue is full 840 */ 841 lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); 842 843 memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); 844 memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); 845 846 ctrl_cmd.eth_dev = eth_dev; 847 ctrl_cmd.cond = 0; 848 849 ctrl_pkt.ncmd.s.cmd = LIO_CMD_VXLAN_PORT_CONFIG; 850 ctrl_pkt.ncmd.s.param1 = udp_tnl->udp_port; 851 ctrl_pkt.ncmd.s.more = LIO_CMD_VXLAN_PORT_DEL; 852 ctrl_pkt.ctrl_cmd = &ctrl_cmd; 853 854 if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { 855 lio_dev_err(lio_dev, "Failed to send VXLAN_PORT_DEL command\n"); 856 return -1; 857 } 858 859 if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) { 860 lio_dev_err(lio_dev, "VXLAN_PORT_DEL command timed out\n"); 861 return -1; 862 } 863 864 return 0; 865 } 866 867 static int 868 lio_dev_vlan_filter_set(struct rte_eth_dev *eth_dev, uint16_t vlan_id, int on) 869 { 870 struct lio_device *lio_dev = LIO_DEV(eth_dev); 871 struct lio_dev_ctrl_cmd ctrl_cmd; 872 struct lio_ctrl_pkt ctrl_pkt; 873 874 if (lio_dev->linfo.vlan_is_admin_assigned) 875 return -EPERM; 876 877 /* flush added to prevent cmd failure 878 * incase the queue is full 879 */ 880 lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); 881 882 memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); 883 memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); 884 885 ctrl_cmd.eth_dev = eth_dev; 886 ctrl_cmd.cond = 0; 887 888 ctrl_pkt.ncmd.s.cmd = on ? 889 LIO_CMD_ADD_VLAN_FILTER : LIO_CMD_DEL_VLAN_FILTER; 890 ctrl_pkt.ncmd.s.param1 = vlan_id; 891 ctrl_pkt.ctrl_cmd = &ctrl_cmd; 892 893 if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { 894 lio_dev_err(lio_dev, "Failed to %s VLAN port\n", 895 on ? "add" : "remove"); 896 return -1; 897 } 898 899 if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) { 900 lio_dev_err(lio_dev, "Command to %s VLAN port timed out\n", 901 on ? "add" : "remove"); 902 return -1; 903 } 904 905 return 0; 906 } 907 908 static uint64_t 909 lio_hweight64(uint64_t w) 910 { 911 uint64_t res = w - ((w >> 1) & 0x5555555555555555ul); 912 913 res = 914 (res & 0x3333333333333333ul) + ((res >> 2) & 0x3333333333333333ul); 915 res = (res + (res >> 4)) & 0x0F0F0F0F0F0F0F0Ful; 916 res = res + (res >> 8); 917 res = res + (res >> 16); 918 919 return (res + (res >> 32)) & 0x00000000000000FFul; 920 } 921 922 static int 923 lio_dev_link_update(struct rte_eth_dev *eth_dev, 924 int wait_to_complete __rte_unused) 925 { 926 struct lio_device *lio_dev = LIO_DEV(eth_dev); 927 struct rte_eth_link link; 928 929 /* Initialize */ 930 memset(&link, 0, sizeof(link)); 931 link.link_status = ETH_LINK_DOWN; 932 link.link_speed = ETH_SPEED_NUM_NONE; 933 link.link_duplex = ETH_LINK_HALF_DUPLEX; 934 link.link_autoneg = ETH_LINK_AUTONEG; 935 936 /* Return what we found */ 937 if (lio_dev->linfo.link.s.link_up == 0) { 938 /* Interface is down */ 939 return rte_eth_linkstatus_set(eth_dev, &link); 940 } 941 942 link.link_status = ETH_LINK_UP; /* Interface is up */ 943 link.link_duplex = ETH_LINK_FULL_DUPLEX; 944 switch (lio_dev->linfo.link.s.speed) { 945 case LIO_LINK_SPEED_10000: 946 link.link_speed = ETH_SPEED_NUM_10G; 947 break; 948 case LIO_LINK_SPEED_25000: 949 link.link_speed = ETH_SPEED_NUM_25G; 950 break; 951 default: 952 link.link_speed = ETH_SPEED_NUM_NONE; 953 link.link_duplex = ETH_LINK_HALF_DUPLEX; 954 } 955 956 return rte_eth_linkstatus_set(eth_dev, &link); 957 } 958 959 /** 960 * \brief Net device enable, disable allmulticast 961 * @param eth_dev Pointer to the structure rte_eth_dev 962 */ 963 static void 964 lio_change_dev_flag(struct rte_eth_dev *eth_dev) 965 { 966 struct lio_device *lio_dev = LIO_DEV(eth_dev); 967 struct lio_dev_ctrl_cmd ctrl_cmd; 968 struct lio_ctrl_pkt ctrl_pkt; 969 970 /* flush added to prevent cmd failure 971 * incase the queue is full 972 */ 973 lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); 974 975 memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); 976 memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); 977 978 ctrl_cmd.eth_dev = eth_dev; 979 ctrl_cmd.cond = 0; 980 981 /* Create a ctrl pkt command to be sent to core app. */ 982 ctrl_pkt.ncmd.s.cmd = LIO_CMD_CHANGE_DEVFLAGS; 983 ctrl_pkt.ncmd.s.param1 = lio_dev->ifflags; 984 ctrl_pkt.ctrl_cmd = &ctrl_cmd; 985 986 if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { 987 lio_dev_err(lio_dev, "Failed to send change flag message\n"); 988 return; 989 } 990 991 if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) 992 lio_dev_err(lio_dev, "Change dev flag command timed out\n"); 993 } 994 995 static void 996 lio_dev_promiscuous_enable(struct rte_eth_dev *eth_dev) 997 { 998 struct lio_device *lio_dev = LIO_DEV(eth_dev); 999 1000 if (strcmp(lio_dev->firmware_version, LIO_VF_TRUST_MIN_VERSION) < 0) { 1001 lio_dev_err(lio_dev, "Require firmware version >= %s\n", 1002 LIO_VF_TRUST_MIN_VERSION); 1003 return; 1004 } 1005 1006 if (!lio_dev->intf_open) { 1007 lio_dev_err(lio_dev, "Port %d down, can't enable promiscuous\n", 1008 lio_dev->port_id); 1009 return; 1010 } 1011 1012 lio_dev->ifflags |= LIO_IFFLAG_PROMISC; 1013 lio_change_dev_flag(eth_dev); 1014 } 1015 1016 static void 1017 lio_dev_promiscuous_disable(struct rte_eth_dev *eth_dev) 1018 { 1019 struct lio_device *lio_dev = LIO_DEV(eth_dev); 1020 1021 if (strcmp(lio_dev->firmware_version, LIO_VF_TRUST_MIN_VERSION) < 0) { 1022 lio_dev_err(lio_dev, "Require firmware version >= %s\n", 1023 LIO_VF_TRUST_MIN_VERSION); 1024 return; 1025 } 1026 1027 if (!lio_dev->intf_open) { 1028 lio_dev_err(lio_dev, "Port %d down, can't disable promiscuous\n", 1029 lio_dev->port_id); 1030 return; 1031 } 1032 1033 lio_dev->ifflags &= ~LIO_IFFLAG_PROMISC; 1034 lio_change_dev_flag(eth_dev); 1035 } 1036 1037 static void 1038 lio_dev_allmulticast_enable(struct rte_eth_dev *eth_dev) 1039 { 1040 struct lio_device *lio_dev = LIO_DEV(eth_dev); 1041 1042 if (!lio_dev->intf_open) { 1043 lio_dev_err(lio_dev, "Port %d down, can't enable multicast\n", 1044 lio_dev->port_id); 1045 return; 1046 } 1047 1048 lio_dev->ifflags |= LIO_IFFLAG_ALLMULTI; 1049 lio_change_dev_flag(eth_dev); 1050 } 1051 1052 static void 1053 lio_dev_allmulticast_disable(struct rte_eth_dev *eth_dev) 1054 { 1055 struct lio_device *lio_dev = LIO_DEV(eth_dev); 1056 1057 if (!lio_dev->intf_open) { 1058 lio_dev_err(lio_dev, "Port %d down, can't disable multicast\n", 1059 lio_dev->port_id); 1060 return; 1061 } 1062 1063 lio_dev->ifflags &= ~LIO_IFFLAG_ALLMULTI; 1064 lio_change_dev_flag(eth_dev); 1065 } 1066 1067 static void 1068 lio_dev_rss_configure(struct rte_eth_dev *eth_dev) 1069 { 1070 struct lio_device *lio_dev = LIO_DEV(eth_dev); 1071 struct lio_rss_ctx *rss_state = &lio_dev->rss_state; 1072 struct rte_eth_rss_reta_entry64 reta_conf[8]; 1073 struct rte_eth_rss_conf rss_conf; 1074 uint16_t i; 1075 1076 /* Configure the RSS key and the RSS protocols used to compute 1077 * the RSS hash of input packets. 1078 */ 1079 rss_conf = eth_dev->data->dev_conf.rx_adv_conf.rss_conf; 1080 if ((rss_conf.rss_hf & LIO_RSS_OFFLOAD_ALL) == 0) { 1081 rss_state->hash_disable = 1; 1082 lio_dev_rss_hash_update(eth_dev, &rss_conf); 1083 return; 1084 } 1085 1086 if (rss_conf.rss_key == NULL) 1087 rss_conf.rss_key = lio_rss_key; /* Default hash key */ 1088 1089 lio_dev_rss_hash_update(eth_dev, &rss_conf); 1090 1091 memset(reta_conf, 0, sizeof(reta_conf)); 1092 for (i = 0; i < LIO_RSS_MAX_TABLE_SZ; i++) { 1093 uint8_t q_idx, conf_idx, reta_idx; 1094 1095 q_idx = (uint8_t)((eth_dev->data->nb_rx_queues > 1) ? 1096 i % eth_dev->data->nb_rx_queues : 0); 1097 conf_idx = i / RTE_RETA_GROUP_SIZE; 1098 reta_idx = i % RTE_RETA_GROUP_SIZE; 1099 reta_conf[conf_idx].reta[reta_idx] = q_idx; 1100 reta_conf[conf_idx].mask |= ((uint64_t)1 << reta_idx); 1101 } 1102 1103 lio_dev_rss_reta_update(eth_dev, reta_conf, LIO_RSS_MAX_TABLE_SZ); 1104 } 1105 1106 static void 1107 lio_dev_mq_rx_configure(struct rte_eth_dev *eth_dev) 1108 { 1109 struct lio_device *lio_dev = LIO_DEV(eth_dev); 1110 struct lio_rss_ctx *rss_state = &lio_dev->rss_state; 1111 struct rte_eth_rss_conf rss_conf; 1112 1113 switch (eth_dev->data->dev_conf.rxmode.mq_mode) { 1114 case ETH_MQ_RX_RSS: 1115 lio_dev_rss_configure(eth_dev); 1116 break; 1117 case ETH_MQ_RX_NONE: 1118 /* if mq_mode is none, disable rss mode. */ 1119 default: 1120 memset(&rss_conf, 0, sizeof(rss_conf)); 1121 rss_state->hash_disable = 1; 1122 lio_dev_rss_hash_update(eth_dev, &rss_conf); 1123 } 1124 } 1125 1126 /** 1127 * Setup our receive queue/ringbuffer. This is the 1128 * queue the Octeon uses to send us packets and 1129 * responses. We are given a memory pool for our 1130 * packet buffers that are used to populate the receive 1131 * queue. 1132 * 1133 * @param eth_dev 1134 * Pointer to the structure rte_eth_dev 1135 * @param q_no 1136 * Queue number 1137 * @param num_rx_descs 1138 * Number of entries in the queue 1139 * @param socket_id 1140 * Where to allocate memory 1141 * @param rx_conf 1142 * Pointer to the struction rte_eth_rxconf 1143 * @param mp 1144 * Pointer to the packet pool 1145 * 1146 * @return 1147 * - On success, return 0 1148 * - On failure, return -1 1149 */ 1150 static int 1151 lio_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t q_no, 1152 uint16_t num_rx_descs, unsigned int socket_id, 1153 const struct rte_eth_rxconf *rx_conf __rte_unused, 1154 struct rte_mempool *mp) 1155 { 1156 struct lio_device *lio_dev = LIO_DEV(eth_dev); 1157 struct rte_pktmbuf_pool_private *mbp_priv; 1158 uint32_t fw_mapped_oq; 1159 uint16_t buf_size; 1160 1161 if (q_no >= lio_dev->nb_rx_queues) { 1162 lio_dev_err(lio_dev, "Invalid rx queue number %u\n", q_no); 1163 return -EINVAL; 1164 } 1165 1166 lio_dev_dbg(lio_dev, "setting up rx queue %u\n", q_no); 1167 1168 fw_mapped_oq = lio_dev->linfo.rxpciq[q_no].s.q_no; 1169 1170 /* Free previous allocation if any */ 1171 if (eth_dev->data->rx_queues[q_no] != NULL) { 1172 lio_dev_rx_queue_release(eth_dev->data->rx_queues[q_no]); 1173 eth_dev->data->rx_queues[q_no] = NULL; 1174 } 1175 1176 mbp_priv = rte_mempool_get_priv(mp); 1177 buf_size = mbp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM; 1178 1179 if (lio_setup_droq(lio_dev, fw_mapped_oq, num_rx_descs, buf_size, mp, 1180 socket_id)) { 1181 lio_dev_err(lio_dev, "droq allocation failed\n"); 1182 return -1; 1183 } 1184 1185 eth_dev->data->rx_queues[q_no] = lio_dev->droq[fw_mapped_oq]; 1186 1187 return 0; 1188 } 1189 1190 /** 1191 * Release the receive queue/ringbuffer. Called by 1192 * the upper layers. 1193 * 1194 * @param rxq 1195 * Opaque pointer to the receive queue to release 1196 * 1197 * @return 1198 * - nothing 1199 */ 1200 void 1201 lio_dev_rx_queue_release(void *rxq) 1202 { 1203 struct lio_droq *droq = rxq; 1204 int oq_no; 1205 1206 if (droq) { 1207 oq_no = droq->q_no; 1208 lio_delete_droq_queue(droq->lio_dev, oq_no); 1209 } 1210 } 1211 1212 /** 1213 * Allocate and initialize SW ring. Initialize associated HW registers. 1214 * 1215 * @param eth_dev 1216 * Pointer to structure rte_eth_dev 1217 * 1218 * @param q_no 1219 * Queue number 1220 * 1221 * @param num_tx_descs 1222 * Number of ringbuffer descriptors 1223 * 1224 * @param socket_id 1225 * NUMA socket id, used for memory allocations 1226 * 1227 * @param tx_conf 1228 * Pointer to the structure rte_eth_txconf 1229 * 1230 * @return 1231 * - On success, return 0 1232 * - On failure, return -errno value 1233 */ 1234 static int 1235 lio_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t q_no, 1236 uint16_t num_tx_descs, unsigned int socket_id, 1237 const struct rte_eth_txconf *tx_conf __rte_unused) 1238 { 1239 struct lio_device *lio_dev = LIO_DEV(eth_dev); 1240 int fw_mapped_iq = lio_dev->linfo.txpciq[q_no].s.q_no; 1241 int retval; 1242 1243 if (q_no >= lio_dev->nb_tx_queues) { 1244 lio_dev_err(lio_dev, "Invalid tx queue number %u\n", q_no); 1245 return -EINVAL; 1246 } 1247 1248 lio_dev_dbg(lio_dev, "setting up tx queue %u\n", q_no); 1249 1250 /* Free previous allocation if any */ 1251 if (eth_dev->data->tx_queues[q_no] != NULL) { 1252 lio_dev_tx_queue_release(eth_dev->data->tx_queues[q_no]); 1253 eth_dev->data->tx_queues[q_no] = NULL; 1254 } 1255 1256 retval = lio_setup_iq(lio_dev, q_no, lio_dev->linfo.txpciq[q_no], 1257 num_tx_descs, lio_dev, socket_id); 1258 1259 if (retval) { 1260 lio_dev_err(lio_dev, "Runtime IQ(TxQ) creation failed.\n"); 1261 return retval; 1262 } 1263 1264 retval = lio_setup_sglists(lio_dev, q_no, fw_mapped_iq, 1265 lio_dev->instr_queue[fw_mapped_iq]->nb_desc, 1266 socket_id); 1267 1268 if (retval) { 1269 lio_delete_instruction_queue(lio_dev, fw_mapped_iq); 1270 return retval; 1271 } 1272 1273 eth_dev->data->tx_queues[q_no] = lio_dev->instr_queue[fw_mapped_iq]; 1274 1275 return 0; 1276 } 1277 1278 /** 1279 * Release the transmit queue/ringbuffer. Called by 1280 * the upper layers. 1281 * 1282 * @param txq 1283 * Opaque pointer to the transmit queue to release 1284 * 1285 * @return 1286 * - nothing 1287 */ 1288 void 1289 lio_dev_tx_queue_release(void *txq) 1290 { 1291 struct lio_instr_queue *tq = txq; 1292 uint32_t fw_mapped_iq_no; 1293 1294 1295 if (tq) { 1296 /* Free sg_list */ 1297 lio_delete_sglist(tq); 1298 1299 fw_mapped_iq_no = tq->txpciq.s.q_no; 1300 lio_delete_instruction_queue(tq->lio_dev, fw_mapped_iq_no); 1301 } 1302 } 1303 1304 /** 1305 * Api to check link state. 1306 */ 1307 static void 1308 lio_dev_get_link_status(struct rte_eth_dev *eth_dev) 1309 { 1310 struct lio_device *lio_dev = LIO_DEV(eth_dev); 1311 uint16_t timeout = LIO_MAX_CMD_TIMEOUT; 1312 struct lio_link_status_resp *resp; 1313 union octeon_link_status *ls; 1314 struct lio_soft_command *sc; 1315 uint32_t resp_size; 1316 1317 if (!lio_dev->intf_open) 1318 return; 1319 1320 resp_size = sizeof(struct lio_link_status_resp); 1321 sc = lio_alloc_soft_command(lio_dev, 0, resp_size, 0); 1322 if (sc == NULL) 1323 return; 1324 1325 resp = (struct lio_link_status_resp *)sc->virtrptr; 1326 lio_prepare_soft_command(lio_dev, sc, LIO_OPCODE, 1327 LIO_OPCODE_INFO, 0, 0, 0); 1328 1329 /* Setting wait time in seconds */ 1330 sc->wait_time = LIO_MAX_CMD_TIMEOUT / 1000; 1331 1332 if (lio_send_soft_command(lio_dev, sc) == LIO_IQ_SEND_FAILED) 1333 goto get_status_fail; 1334 1335 while ((*sc->status_word == LIO_COMPLETION_WORD_INIT) && --timeout) { 1336 lio_flush_iq(lio_dev, lio_dev->instr_queue[sc->iq_no]); 1337 rte_delay_ms(1); 1338 } 1339 1340 if (resp->status) 1341 goto get_status_fail; 1342 1343 ls = &resp->link_info.link; 1344 1345 lio_swap_8B_data((uint64_t *)ls, sizeof(union octeon_link_status) >> 3); 1346 1347 if (lio_dev->linfo.link.link_status64 != ls->link_status64) { 1348 if (ls->s.mtu < eth_dev->data->mtu) { 1349 lio_dev_info(lio_dev, "Lowered VF MTU to %d as PF MTU dropped\n", 1350 ls->s.mtu); 1351 eth_dev->data->mtu = ls->s.mtu; 1352 } 1353 lio_dev->linfo.link.link_status64 = ls->link_status64; 1354 lio_dev_link_update(eth_dev, 0); 1355 } 1356 1357 lio_free_soft_command(sc); 1358 1359 return; 1360 1361 get_status_fail: 1362 lio_free_soft_command(sc); 1363 } 1364 1365 /* This function will be invoked every LSC_TIMEOUT ns (100ms) 1366 * and will update link state if it changes. 1367 */ 1368 static void 1369 lio_sync_link_state_check(void *eth_dev) 1370 { 1371 struct lio_device *lio_dev = 1372 (((struct rte_eth_dev *)eth_dev)->data->dev_private); 1373 1374 if (lio_dev->port_configured) 1375 lio_dev_get_link_status(eth_dev); 1376 1377 /* Schedule periodic link status check. 1378 * Stop check if interface is close and start again while opening. 1379 */ 1380 if (lio_dev->intf_open) 1381 rte_eal_alarm_set(LIO_LSC_TIMEOUT, lio_sync_link_state_check, 1382 eth_dev); 1383 } 1384 1385 static int 1386 lio_dev_start(struct rte_eth_dev *eth_dev) 1387 { 1388 uint16_t mtu; 1389 uint32_t frame_len = eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; 1390 struct lio_device *lio_dev = LIO_DEV(eth_dev); 1391 uint16_t timeout = LIO_MAX_CMD_TIMEOUT; 1392 int ret = 0; 1393 1394 lio_dev_info(lio_dev, "Starting port %d\n", eth_dev->data->port_id); 1395 1396 if (lio_dev->fn_list.enable_io_queues(lio_dev)) 1397 return -1; 1398 1399 if (lio_send_rx_ctrl_cmd(eth_dev, 1)) 1400 return -1; 1401 1402 /* Ready for link status updates */ 1403 lio_dev->intf_open = 1; 1404 rte_mb(); 1405 1406 /* Configure RSS if device configured with multiple RX queues. */ 1407 lio_dev_mq_rx_configure(eth_dev); 1408 1409 /* Before update the link info, 1410 * must set linfo.link.link_status64 to 0. 1411 */ 1412 lio_dev->linfo.link.link_status64 = 0; 1413 1414 /* start polling for lsc */ 1415 ret = rte_eal_alarm_set(LIO_LSC_TIMEOUT, 1416 lio_sync_link_state_check, 1417 eth_dev); 1418 if (ret) { 1419 lio_dev_err(lio_dev, 1420 "link state check handler creation failed\n"); 1421 goto dev_lsc_handle_error; 1422 } 1423 1424 while ((lio_dev->linfo.link.link_status64 == 0) && (--timeout)) 1425 rte_delay_ms(1); 1426 1427 if (lio_dev->linfo.link.link_status64 == 0) { 1428 ret = -1; 1429 goto dev_mtu_set_error; 1430 } 1431 1432 mtu = (uint16_t)(frame_len - RTE_ETHER_HDR_LEN - RTE_ETHER_CRC_LEN); 1433 if (mtu < RTE_ETHER_MIN_MTU) 1434 mtu = RTE_ETHER_MIN_MTU; 1435 1436 if (eth_dev->data->mtu != mtu) { 1437 ret = lio_dev_mtu_set(eth_dev, mtu); 1438 if (ret) 1439 goto dev_mtu_set_error; 1440 } 1441 1442 return 0; 1443 1444 dev_mtu_set_error: 1445 rte_eal_alarm_cancel(lio_sync_link_state_check, eth_dev); 1446 1447 dev_lsc_handle_error: 1448 lio_dev->intf_open = 0; 1449 lio_send_rx_ctrl_cmd(eth_dev, 0); 1450 1451 return ret; 1452 } 1453 1454 /* Stop device and disable input/output functions */ 1455 static void 1456 lio_dev_stop(struct rte_eth_dev *eth_dev) 1457 { 1458 struct lio_device *lio_dev = LIO_DEV(eth_dev); 1459 1460 lio_dev_info(lio_dev, "Stopping port %d\n", eth_dev->data->port_id); 1461 lio_dev->intf_open = 0; 1462 rte_mb(); 1463 1464 /* Cancel callback if still running. */ 1465 rte_eal_alarm_cancel(lio_sync_link_state_check, eth_dev); 1466 1467 lio_send_rx_ctrl_cmd(eth_dev, 0); 1468 1469 lio_wait_for_instr_fetch(lio_dev); 1470 1471 /* Clear recorded link status */ 1472 lio_dev->linfo.link.link_status64 = 0; 1473 } 1474 1475 static int 1476 lio_dev_set_link_up(struct rte_eth_dev *eth_dev) 1477 { 1478 struct lio_device *lio_dev = LIO_DEV(eth_dev); 1479 1480 if (!lio_dev->intf_open) { 1481 lio_dev_info(lio_dev, "Port is stopped, Start the port first\n"); 1482 return 0; 1483 } 1484 1485 if (lio_dev->linfo.link.s.link_up) { 1486 lio_dev_info(lio_dev, "Link is already UP\n"); 1487 return 0; 1488 } 1489 1490 if (lio_send_rx_ctrl_cmd(eth_dev, 1)) { 1491 lio_dev_err(lio_dev, "Unable to set Link UP\n"); 1492 return -1; 1493 } 1494 1495 lio_dev->linfo.link.s.link_up = 1; 1496 eth_dev->data->dev_link.link_status = ETH_LINK_UP; 1497 1498 return 0; 1499 } 1500 1501 static int 1502 lio_dev_set_link_down(struct rte_eth_dev *eth_dev) 1503 { 1504 struct lio_device *lio_dev = LIO_DEV(eth_dev); 1505 1506 if (!lio_dev->intf_open) { 1507 lio_dev_info(lio_dev, "Port is stopped, Start the port first\n"); 1508 return 0; 1509 } 1510 1511 if (!lio_dev->linfo.link.s.link_up) { 1512 lio_dev_info(lio_dev, "Link is already DOWN\n"); 1513 return 0; 1514 } 1515 1516 lio_dev->linfo.link.s.link_up = 0; 1517 eth_dev->data->dev_link.link_status = ETH_LINK_DOWN; 1518 1519 if (lio_send_rx_ctrl_cmd(eth_dev, 0)) { 1520 lio_dev->linfo.link.s.link_up = 1; 1521 eth_dev->data->dev_link.link_status = ETH_LINK_UP; 1522 lio_dev_err(lio_dev, "Unable to set Link Down\n"); 1523 return -1; 1524 } 1525 1526 return 0; 1527 } 1528 1529 /** 1530 * Reset and stop the device. This occurs on the first 1531 * call to this routine. Subsequent calls will simply 1532 * return. NB: This will require the NIC to be rebooted. 1533 * 1534 * @param eth_dev 1535 * Pointer to the structure rte_eth_dev 1536 * 1537 * @return 1538 * - nothing 1539 */ 1540 static void 1541 lio_dev_close(struct rte_eth_dev *eth_dev) 1542 { 1543 struct lio_device *lio_dev = LIO_DEV(eth_dev); 1544 1545 lio_dev_info(lio_dev, "closing port %d\n", eth_dev->data->port_id); 1546 1547 if (lio_dev->intf_open) 1548 lio_dev_stop(eth_dev); 1549 1550 /* Reset ioq regs */ 1551 lio_dev->fn_list.setup_device_regs(lio_dev); 1552 1553 if (lio_dev->pci_dev->kdrv == RTE_KDRV_IGB_UIO) { 1554 cn23xx_vf_ask_pf_to_do_flr(lio_dev); 1555 rte_delay_ms(LIO_PCI_FLR_WAIT); 1556 } 1557 1558 /* lio_free_mbox */ 1559 lio_dev->fn_list.free_mbox(lio_dev); 1560 1561 /* Free glist resources */ 1562 rte_free(lio_dev->glist_head); 1563 rte_free(lio_dev->glist_lock); 1564 lio_dev->glist_head = NULL; 1565 lio_dev->glist_lock = NULL; 1566 1567 lio_dev->port_configured = 0; 1568 1569 /* Delete all queues */ 1570 lio_dev_clear_queues(eth_dev); 1571 } 1572 1573 /** 1574 * Enable tunnel rx checksum verification from firmware. 1575 */ 1576 static void 1577 lio_enable_hw_tunnel_rx_checksum(struct rte_eth_dev *eth_dev) 1578 { 1579 struct lio_device *lio_dev = LIO_DEV(eth_dev); 1580 struct lio_dev_ctrl_cmd ctrl_cmd; 1581 struct lio_ctrl_pkt ctrl_pkt; 1582 1583 /* flush added to prevent cmd failure 1584 * incase the queue is full 1585 */ 1586 lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); 1587 1588 memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); 1589 memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); 1590 1591 ctrl_cmd.eth_dev = eth_dev; 1592 ctrl_cmd.cond = 0; 1593 1594 ctrl_pkt.ncmd.s.cmd = LIO_CMD_TNL_RX_CSUM_CTL; 1595 ctrl_pkt.ncmd.s.param1 = LIO_CMD_RXCSUM_ENABLE; 1596 ctrl_pkt.ctrl_cmd = &ctrl_cmd; 1597 1598 if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { 1599 lio_dev_err(lio_dev, "Failed to send TNL_RX_CSUM command\n"); 1600 return; 1601 } 1602 1603 if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) 1604 lio_dev_err(lio_dev, "TNL_RX_CSUM command timed out\n"); 1605 } 1606 1607 /** 1608 * Enable checksum calculation for inner packet in a tunnel. 1609 */ 1610 static void 1611 lio_enable_hw_tunnel_tx_checksum(struct rte_eth_dev *eth_dev) 1612 { 1613 struct lio_device *lio_dev = LIO_DEV(eth_dev); 1614 struct lio_dev_ctrl_cmd ctrl_cmd; 1615 struct lio_ctrl_pkt ctrl_pkt; 1616 1617 /* flush added to prevent cmd failure 1618 * incase the queue is full 1619 */ 1620 lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); 1621 1622 memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); 1623 memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); 1624 1625 ctrl_cmd.eth_dev = eth_dev; 1626 ctrl_cmd.cond = 0; 1627 1628 ctrl_pkt.ncmd.s.cmd = LIO_CMD_TNL_TX_CSUM_CTL; 1629 ctrl_pkt.ncmd.s.param1 = LIO_CMD_TXCSUM_ENABLE; 1630 ctrl_pkt.ctrl_cmd = &ctrl_cmd; 1631 1632 if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { 1633 lio_dev_err(lio_dev, "Failed to send TNL_TX_CSUM command\n"); 1634 return; 1635 } 1636 1637 if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) 1638 lio_dev_err(lio_dev, "TNL_TX_CSUM command timed out\n"); 1639 } 1640 1641 static int 1642 lio_send_queue_count_update(struct rte_eth_dev *eth_dev, int num_txq, 1643 int num_rxq) 1644 { 1645 struct lio_device *lio_dev = LIO_DEV(eth_dev); 1646 struct lio_dev_ctrl_cmd ctrl_cmd; 1647 struct lio_ctrl_pkt ctrl_pkt; 1648 1649 if (strcmp(lio_dev->firmware_version, LIO_Q_RECONF_MIN_VERSION) < 0) { 1650 lio_dev_err(lio_dev, "Require firmware version >= %s\n", 1651 LIO_Q_RECONF_MIN_VERSION); 1652 return -ENOTSUP; 1653 } 1654 1655 /* flush added to prevent cmd failure 1656 * incase the queue is full 1657 */ 1658 lio_flush_iq(lio_dev, lio_dev->instr_queue[0]); 1659 1660 memset(&ctrl_pkt, 0, sizeof(struct lio_ctrl_pkt)); 1661 memset(&ctrl_cmd, 0, sizeof(struct lio_dev_ctrl_cmd)); 1662 1663 ctrl_cmd.eth_dev = eth_dev; 1664 ctrl_cmd.cond = 0; 1665 1666 ctrl_pkt.ncmd.s.cmd = LIO_CMD_QUEUE_COUNT_CTL; 1667 ctrl_pkt.ncmd.s.param1 = num_txq; 1668 ctrl_pkt.ncmd.s.param2 = num_rxq; 1669 ctrl_pkt.ctrl_cmd = &ctrl_cmd; 1670 1671 if (lio_send_ctrl_pkt(lio_dev, &ctrl_pkt)) { 1672 lio_dev_err(lio_dev, "Failed to send queue count control command\n"); 1673 return -1; 1674 } 1675 1676 if (lio_wait_for_ctrl_cmd(lio_dev, &ctrl_cmd)) { 1677 lio_dev_err(lio_dev, "Queue count control command timed out\n"); 1678 return -1; 1679 } 1680 1681 return 0; 1682 } 1683 1684 static int 1685 lio_reconf_queues(struct rte_eth_dev *eth_dev, int num_txq, int num_rxq) 1686 { 1687 struct lio_device *lio_dev = LIO_DEV(eth_dev); 1688 1689 if (lio_dev->nb_rx_queues != num_rxq || 1690 lio_dev->nb_tx_queues != num_txq) { 1691 if (lio_send_queue_count_update(eth_dev, num_txq, num_rxq)) 1692 return -1; 1693 lio_dev->nb_rx_queues = num_rxq; 1694 lio_dev->nb_tx_queues = num_txq; 1695 } 1696 1697 if (lio_dev->intf_open) 1698 lio_dev_stop(eth_dev); 1699 1700 /* Reset ioq registers */ 1701 if (lio_dev->fn_list.setup_device_regs(lio_dev)) { 1702 lio_dev_err(lio_dev, "Failed to configure device registers\n"); 1703 return -1; 1704 } 1705 1706 return 0; 1707 } 1708 1709 static int 1710 lio_dev_configure(struct rte_eth_dev *eth_dev) 1711 { 1712 struct lio_device *lio_dev = LIO_DEV(eth_dev); 1713 uint16_t timeout = LIO_MAX_CMD_TIMEOUT; 1714 int retval, num_iqueues, num_oqueues; 1715 uint8_t mac[RTE_ETHER_ADDR_LEN], i; 1716 struct lio_if_cfg_resp *resp; 1717 struct lio_soft_command *sc; 1718 union lio_if_cfg if_cfg; 1719 uint32_t resp_size; 1720 1721 PMD_INIT_FUNC_TRACE(); 1722 1723 /* Inform firmware about change in number of queues to use. 1724 * Disable IO queues and reset registers for re-configuration. 1725 */ 1726 if (lio_dev->port_configured) 1727 return lio_reconf_queues(eth_dev, 1728 eth_dev->data->nb_tx_queues, 1729 eth_dev->data->nb_rx_queues); 1730 1731 lio_dev->nb_rx_queues = eth_dev->data->nb_rx_queues; 1732 lio_dev->nb_tx_queues = eth_dev->data->nb_tx_queues; 1733 1734 /* Set max number of queues which can be re-configured. */ 1735 lio_dev->max_rx_queues = eth_dev->data->nb_rx_queues; 1736 lio_dev->max_tx_queues = eth_dev->data->nb_tx_queues; 1737 1738 resp_size = sizeof(struct lio_if_cfg_resp); 1739 sc = lio_alloc_soft_command(lio_dev, 0, resp_size, 0); 1740 if (sc == NULL) 1741 return -ENOMEM; 1742 1743 resp = (struct lio_if_cfg_resp *)sc->virtrptr; 1744 1745 /* Firmware doesn't have capability to reconfigure the queues, 1746 * Claim all queues, and use as many required 1747 */ 1748 if_cfg.if_cfg64 = 0; 1749 if_cfg.s.num_iqueues = lio_dev->nb_tx_queues; 1750 if_cfg.s.num_oqueues = lio_dev->nb_rx_queues; 1751 if_cfg.s.base_queue = 0; 1752 1753 if_cfg.s.gmx_port_id = lio_dev->pf_num; 1754 1755 lio_prepare_soft_command(lio_dev, sc, LIO_OPCODE, 1756 LIO_OPCODE_IF_CFG, 0, 1757 if_cfg.if_cfg64, 0); 1758 1759 /* Setting wait time in seconds */ 1760 sc->wait_time = LIO_MAX_CMD_TIMEOUT / 1000; 1761 1762 retval = lio_send_soft_command(lio_dev, sc); 1763 if (retval == LIO_IQ_SEND_FAILED) { 1764 lio_dev_err(lio_dev, "iq/oq config failed status: %x\n", 1765 retval); 1766 /* Soft instr is freed by driver in case of failure. */ 1767 goto nic_config_fail; 1768 } 1769 1770 /* Sleep on a wait queue till the cond flag indicates that the 1771 * response arrived or timed-out. 1772 */ 1773 while ((*sc->status_word == LIO_COMPLETION_WORD_INIT) && --timeout) { 1774 lio_flush_iq(lio_dev, lio_dev->instr_queue[sc->iq_no]); 1775 lio_process_ordered_list(lio_dev); 1776 rte_delay_ms(1); 1777 } 1778 1779 retval = resp->status; 1780 if (retval) { 1781 lio_dev_err(lio_dev, "iq/oq config failed\n"); 1782 goto nic_config_fail; 1783 } 1784 1785 strlcpy(lio_dev->firmware_version, 1786 resp->cfg_info.lio_firmware_version, LIO_FW_VERSION_LENGTH); 1787 1788 lio_swap_8B_data((uint64_t *)(&resp->cfg_info), 1789 sizeof(struct octeon_if_cfg_info) >> 3); 1790 1791 num_iqueues = lio_hweight64(resp->cfg_info.iqmask); 1792 num_oqueues = lio_hweight64(resp->cfg_info.oqmask); 1793 1794 if (!(num_iqueues) || !(num_oqueues)) { 1795 lio_dev_err(lio_dev, 1796 "Got bad iqueues (%016lx) or oqueues (%016lx) from firmware.\n", 1797 (unsigned long)resp->cfg_info.iqmask, 1798 (unsigned long)resp->cfg_info.oqmask); 1799 goto nic_config_fail; 1800 } 1801 1802 lio_dev_dbg(lio_dev, 1803 "interface %d, iqmask %016lx, oqmask %016lx, numiqueues %d, numoqueues %d\n", 1804 eth_dev->data->port_id, 1805 (unsigned long)resp->cfg_info.iqmask, 1806 (unsigned long)resp->cfg_info.oqmask, 1807 num_iqueues, num_oqueues); 1808 1809 lio_dev->linfo.num_rxpciq = num_oqueues; 1810 lio_dev->linfo.num_txpciq = num_iqueues; 1811 1812 for (i = 0; i < num_oqueues; i++) { 1813 lio_dev->linfo.rxpciq[i].rxpciq64 = 1814 resp->cfg_info.linfo.rxpciq[i].rxpciq64; 1815 lio_dev_dbg(lio_dev, "index %d OQ %d\n", 1816 i, lio_dev->linfo.rxpciq[i].s.q_no); 1817 } 1818 1819 for (i = 0; i < num_iqueues; i++) { 1820 lio_dev->linfo.txpciq[i].txpciq64 = 1821 resp->cfg_info.linfo.txpciq[i].txpciq64; 1822 lio_dev_dbg(lio_dev, "index %d IQ %d\n", 1823 i, lio_dev->linfo.txpciq[i].s.q_no); 1824 } 1825 1826 lio_dev->linfo.hw_addr = resp->cfg_info.linfo.hw_addr; 1827 lio_dev->linfo.gmxport = resp->cfg_info.linfo.gmxport; 1828 lio_dev->linfo.link.link_status64 = 1829 resp->cfg_info.linfo.link.link_status64; 1830 1831 /* 64-bit swap required on LE machines */ 1832 lio_swap_8B_data(&lio_dev->linfo.hw_addr, 1); 1833 for (i = 0; i < RTE_ETHER_ADDR_LEN; i++) 1834 mac[i] = *((uint8_t *)(((uint8_t *)&lio_dev->linfo.hw_addr) + 1835 2 + i)); 1836 1837 /* Copy the permanent MAC address */ 1838 rte_ether_addr_copy((struct rte_ether_addr *)mac, 1839 ð_dev->data->mac_addrs[0]); 1840 1841 /* enable firmware checksum support for tunnel packets */ 1842 lio_enable_hw_tunnel_rx_checksum(eth_dev); 1843 lio_enable_hw_tunnel_tx_checksum(eth_dev); 1844 1845 lio_dev->glist_lock = 1846 rte_zmalloc(NULL, sizeof(*lio_dev->glist_lock) * num_iqueues, 0); 1847 if (lio_dev->glist_lock == NULL) 1848 return -ENOMEM; 1849 1850 lio_dev->glist_head = 1851 rte_zmalloc(NULL, sizeof(*lio_dev->glist_head) * num_iqueues, 1852 0); 1853 if (lio_dev->glist_head == NULL) { 1854 rte_free(lio_dev->glist_lock); 1855 lio_dev->glist_lock = NULL; 1856 return -ENOMEM; 1857 } 1858 1859 lio_dev_link_update(eth_dev, 0); 1860 1861 lio_dev->port_configured = 1; 1862 1863 lio_free_soft_command(sc); 1864 1865 /* Reset ioq regs */ 1866 lio_dev->fn_list.setup_device_regs(lio_dev); 1867 1868 /* Free iq_0 used during init */ 1869 lio_free_instr_queue0(lio_dev); 1870 1871 return 0; 1872 1873 nic_config_fail: 1874 lio_dev_err(lio_dev, "Failed retval %d\n", retval); 1875 lio_free_soft_command(sc); 1876 lio_free_instr_queue0(lio_dev); 1877 1878 return -ENODEV; 1879 } 1880 1881 /* Define our ethernet definitions */ 1882 static const struct eth_dev_ops liovf_eth_dev_ops = { 1883 .dev_configure = lio_dev_configure, 1884 .dev_start = lio_dev_start, 1885 .dev_stop = lio_dev_stop, 1886 .dev_set_link_up = lio_dev_set_link_up, 1887 .dev_set_link_down = lio_dev_set_link_down, 1888 .dev_close = lio_dev_close, 1889 .promiscuous_enable = lio_dev_promiscuous_enable, 1890 .promiscuous_disable = lio_dev_promiscuous_disable, 1891 .allmulticast_enable = lio_dev_allmulticast_enable, 1892 .allmulticast_disable = lio_dev_allmulticast_disable, 1893 .link_update = lio_dev_link_update, 1894 .stats_get = lio_dev_stats_get, 1895 .xstats_get = lio_dev_xstats_get, 1896 .xstats_get_names = lio_dev_xstats_get_names, 1897 .stats_reset = lio_dev_stats_reset, 1898 .xstats_reset = lio_dev_xstats_reset, 1899 .dev_infos_get = lio_dev_info_get, 1900 .vlan_filter_set = lio_dev_vlan_filter_set, 1901 .rx_queue_setup = lio_dev_rx_queue_setup, 1902 .rx_queue_release = lio_dev_rx_queue_release, 1903 .tx_queue_setup = lio_dev_tx_queue_setup, 1904 .tx_queue_release = lio_dev_tx_queue_release, 1905 .reta_update = lio_dev_rss_reta_update, 1906 .reta_query = lio_dev_rss_reta_query, 1907 .rss_hash_conf_get = lio_dev_rss_hash_conf_get, 1908 .rss_hash_update = lio_dev_rss_hash_update, 1909 .udp_tunnel_port_add = lio_dev_udp_tunnel_add, 1910 .udp_tunnel_port_del = lio_dev_udp_tunnel_del, 1911 .mtu_set = lio_dev_mtu_set, 1912 }; 1913 1914 static void 1915 lio_check_pf_hs_response(void *lio_dev) 1916 { 1917 struct lio_device *dev = lio_dev; 1918 1919 /* check till response arrives */ 1920 if (dev->pfvf_hsword.coproc_tics_per_us) 1921 return; 1922 1923 cn23xx_vf_handle_mbox(dev); 1924 1925 rte_eal_alarm_set(1, lio_check_pf_hs_response, lio_dev); 1926 } 1927 1928 /** 1929 * \brief Identify the LIO device and to map the BAR address space 1930 * @param lio_dev lio device 1931 */ 1932 static int 1933 lio_chip_specific_setup(struct lio_device *lio_dev) 1934 { 1935 struct rte_pci_device *pdev = lio_dev->pci_dev; 1936 uint32_t dev_id = pdev->id.device_id; 1937 const char *s; 1938 int ret = 1; 1939 1940 switch (dev_id) { 1941 case LIO_CN23XX_VF_VID: 1942 lio_dev->chip_id = LIO_CN23XX_VF_VID; 1943 ret = cn23xx_vf_setup_device(lio_dev); 1944 s = "CN23XX VF"; 1945 break; 1946 default: 1947 s = "?"; 1948 lio_dev_err(lio_dev, "Unsupported Chip\n"); 1949 } 1950 1951 if (!ret) 1952 lio_dev_info(lio_dev, "DEVICE : %s\n", s); 1953 1954 return ret; 1955 } 1956 1957 static int 1958 lio_first_time_init(struct lio_device *lio_dev, 1959 struct rte_pci_device *pdev) 1960 { 1961 int dpdk_queues; 1962 1963 PMD_INIT_FUNC_TRACE(); 1964 1965 /* set dpdk specific pci device pointer */ 1966 lio_dev->pci_dev = pdev; 1967 1968 /* Identify the LIO type and set device ops */ 1969 if (lio_chip_specific_setup(lio_dev)) { 1970 lio_dev_err(lio_dev, "Chip specific setup failed\n"); 1971 return -1; 1972 } 1973 1974 /* Initialize soft command buffer pool */ 1975 if (lio_setup_sc_buffer_pool(lio_dev)) { 1976 lio_dev_err(lio_dev, "sc buffer pool allocation failed\n"); 1977 return -1; 1978 } 1979 1980 /* Initialize lists to manage the requests of different types that 1981 * arrive from applications for this lio device. 1982 */ 1983 lio_setup_response_list(lio_dev); 1984 1985 if (lio_dev->fn_list.setup_mbox(lio_dev)) { 1986 lio_dev_err(lio_dev, "Mailbox setup failed\n"); 1987 goto error; 1988 } 1989 1990 /* Check PF response */ 1991 lio_check_pf_hs_response((void *)lio_dev); 1992 1993 /* Do handshake and exit if incompatible PF driver */ 1994 if (cn23xx_pfvf_handshake(lio_dev)) 1995 goto error; 1996 1997 /* Request and wait for device reset. */ 1998 if (pdev->kdrv == RTE_KDRV_IGB_UIO) { 1999 cn23xx_vf_ask_pf_to_do_flr(lio_dev); 2000 /* FLR wait time doubled as a precaution. */ 2001 rte_delay_ms(LIO_PCI_FLR_WAIT * 2); 2002 } 2003 2004 if (lio_dev->fn_list.setup_device_regs(lio_dev)) { 2005 lio_dev_err(lio_dev, "Failed to configure device registers\n"); 2006 goto error; 2007 } 2008 2009 if (lio_setup_instr_queue0(lio_dev)) { 2010 lio_dev_err(lio_dev, "Failed to setup instruction queue 0\n"); 2011 goto error; 2012 } 2013 2014 dpdk_queues = (int)lio_dev->sriov_info.rings_per_vf; 2015 2016 lio_dev->max_tx_queues = dpdk_queues; 2017 lio_dev->max_rx_queues = dpdk_queues; 2018 2019 /* Enable input and output queues for this device */ 2020 if (lio_dev->fn_list.enable_io_queues(lio_dev)) 2021 goto error; 2022 2023 return 0; 2024 2025 error: 2026 lio_free_sc_buffer_pool(lio_dev); 2027 if (lio_dev->mbox[0]) 2028 lio_dev->fn_list.free_mbox(lio_dev); 2029 if (lio_dev->instr_queue[0]) 2030 lio_free_instr_queue0(lio_dev); 2031 2032 return -1; 2033 } 2034 2035 static int 2036 lio_eth_dev_uninit(struct rte_eth_dev *eth_dev) 2037 { 2038 struct lio_device *lio_dev = LIO_DEV(eth_dev); 2039 2040 PMD_INIT_FUNC_TRACE(); 2041 2042 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 2043 return 0; 2044 2045 /* lio_free_sc_buffer_pool */ 2046 lio_free_sc_buffer_pool(lio_dev); 2047 2048 eth_dev->dev_ops = NULL; 2049 eth_dev->rx_pkt_burst = NULL; 2050 eth_dev->tx_pkt_burst = NULL; 2051 2052 return 0; 2053 } 2054 2055 static int 2056 lio_eth_dev_init(struct rte_eth_dev *eth_dev) 2057 { 2058 struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(eth_dev); 2059 struct lio_device *lio_dev = LIO_DEV(eth_dev); 2060 2061 PMD_INIT_FUNC_TRACE(); 2062 2063 eth_dev->rx_pkt_burst = &lio_dev_recv_pkts; 2064 eth_dev->tx_pkt_burst = &lio_dev_xmit_pkts; 2065 2066 /* Primary does the initialization. */ 2067 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 2068 return 0; 2069 2070 rte_eth_copy_pci_info(eth_dev, pdev); 2071 2072 if (pdev->mem_resource[0].addr) { 2073 lio_dev->hw_addr = pdev->mem_resource[0].addr; 2074 } else { 2075 PMD_INIT_LOG(ERR, "ERROR: Failed to map BAR0\n"); 2076 return -ENODEV; 2077 } 2078 2079 lio_dev->eth_dev = eth_dev; 2080 /* set lio device print string */ 2081 snprintf(lio_dev->dev_string, sizeof(lio_dev->dev_string), 2082 "%s[%02x:%02x.%x]", pdev->driver->driver.name, 2083 pdev->addr.bus, pdev->addr.devid, pdev->addr.function); 2084 2085 lio_dev->port_id = eth_dev->data->port_id; 2086 2087 if (lio_first_time_init(lio_dev, pdev)) { 2088 lio_dev_err(lio_dev, "Device init failed\n"); 2089 return -EINVAL; 2090 } 2091 2092 eth_dev->dev_ops = &liovf_eth_dev_ops; 2093 eth_dev->data->mac_addrs = rte_zmalloc("lio", RTE_ETHER_ADDR_LEN, 0); 2094 if (eth_dev->data->mac_addrs == NULL) { 2095 lio_dev_err(lio_dev, 2096 "MAC addresses memory allocation failed\n"); 2097 eth_dev->dev_ops = NULL; 2098 eth_dev->rx_pkt_burst = NULL; 2099 eth_dev->tx_pkt_burst = NULL; 2100 return -ENOMEM; 2101 } 2102 2103 rte_atomic64_set(&lio_dev->status, LIO_DEV_RUNNING); 2104 rte_wmb(); 2105 2106 lio_dev->port_configured = 0; 2107 /* Always allow unicast packets */ 2108 lio_dev->ifflags |= LIO_IFFLAG_UNICAST; 2109 2110 return 0; 2111 } 2112 2113 static int 2114 lio_eth_dev_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 2115 struct rte_pci_device *pci_dev) 2116 { 2117 return rte_eth_dev_pci_generic_probe(pci_dev, sizeof(struct lio_device), 2118 lio_eth_dev_init); 2119 } 2120 2121 static int 2122 lio_eth_dev_pci_remove(struct rte_pci_device *pci_dev) 2123 { 2124 return rte_eth_dev_pci_generic_remove(pci_dev, 2125 lio_eth_dev_uninit); 2126 } 2127 2128 /* Set of PCI devices this driver supports */ 2129 static const struct rte_pci_id pci_id_liovf_map[] = { 2130 { RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, LIO_CN23XX_VF_VID) }, 2131 { .vendor_id = 0, /* sentinel */ } 2132 }; 2133 2134 static struct rte_pci_driver rte_liovf_pmd = { 2135 .id_table = pci_id_liovf_map, 2136 .drv_flags = RTE_PCI_DRV_NEED_MAPPING, 2137 .probe = lio_eth_dev_pci_probe, 2138 .remove = lio_eth_dev_pci_remove, 2139 }; 2140 2141 RTE_PMD_REGISTER_PCI(net_liovf, rte_liovf_pmd); 2142 RTE_PMD_REGISTER_PCI_TABLE(net_liovf, pci_id_liovf_map); 2143 RTE_PMD_REGISTER_KMOD_DEP(net_liovf, "* igb_uio | vfio-pci"); 2144 2145 RTE_INIT(lio_init_log) 2146 { 2147 lio_logtype_init = rte_log_register("pmd.net.liquidio.init"); 2148 if (lio_logtype_init >= 0) 2149 rte_log_set_level(lio_logtype_init, RTE_LOG_NOTICE); 2150 lio_logtype_driver = rte_log_register("pmd.net.liquidio.driver"); 2151 if (lio_logtype_driver >= 0) 2152 rte_log_set_level(lio_logtype_driver, RTE_LOG_NOTICE); 2153 } 2154