1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 #include <stdlib.h> 34 #include <netinet/in.h> 35 36 #include <rte_mbuf.h> 37 #include <rte_malloc.h> 38 #include <rte_ethdev.h> 39 #include <rte_ethdev_vdev.h> 40 #include <rte_tcp.h> 41 #include <rte_udp.h> 42 #include <rte_ip.h> 43 #include <rte_ip_frag.h> 44 #include <rte_devargs.h> 45 #include <rte_kvargs.h> 46 #include <rte_bus_vdev.h> 47 #include <rte_alarm.h> 48 #include <rte_cycles.h> 49 50 #include "rte_eth_bond.h" 51 #include "rte_eth_bond_private.h" 52 #include "rte_eth_bond_8023ad_private.h" 53 54 #define REORDER_PERIOD_MS 10 55 #define DEFAULT_POLLING_INTERVAL_10_MS (10) 56 57 #define HASH_L4_PORTS(h) ((h)->src_port ^ (h)->dst_port) 58 59 /* Table for statistics in mode 5 TLB */ 60 static uint64_t tlb_last_obytets[RTE_MAX_ETHPORTS]; 61 62 static inline size_t 63 get_vlan_offset(struct ether_hdr *eth_hdr, uint16_t *proto) 64 { 65 size_t vlan_offset = 0; 66 67 if (rte_cpu_to_be_16(ETHER_TYPE_VLAN) == *proto) { 68 struct vlan_hdr *vlan_hdr = (struct vlan_hdr *)(eth_hdr + 1); 69 70 vlan_offset = sizeof(struct vlan_hdr); 71 *proto = vlan_hdr->eth_proto; 72 73 if (rte_cpu_to_be_16(ETHER_TYPE_VLAN) == *proto) { 74 vlan_hdr = vlan_hdr + 1; 75 *proto = vlan_hdr->eth_proto; 76 vlan_offset += sizeof(struct vlan_hdr); 77 } 78 } 79 return vlan_offset; 80 } 81 82 static uint16_t 83 bond_ethdev_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) 84 { 85 struct bond_dev_private *internals; 86 87 uint16_t num_rx_slave = 0; 88 uint16_t num_rx_total = 0; 89 90 int i; 91 92 /* Cast to structure, containing bonded device's port id and queue id */ 93 struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *)queue; 94 95 internals = bd_rx_q->dev_private; 96 97 98 for (i = 0; i < internals->active_slave_count && nb_pkts; i++) { 99 /* Offset of pointer to *bufs increases as packets are received 100 * from other slaves */ 101 num_rx_slave = rte_eth_rx_burst(internals->active_slaves[i], 102 bd_rx_q->queue_id, bufs + num_rx_total, nb_pkts); 103 if (num_rx_slave) { 104 num_rx_total += num_rx_slave; 105 nb_pkts -= num_rx_slave; 106 } 107 } 108 109 return num_rx_total; 110 } 111 112 static uint16_t 113 bond_ethdev_rx_burst_active_backup(void *queue, struct rte_mbuf **bufs, 114 uint16_t nb_pkts) 115 { 116 struct bond_dev_private *internals; 117 118 /* Cast to structure, containing bonded device's port id and queue id */ 119 struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *)queue; 120 121 internals = bd_rx_q->dev_private; 122 123 return rte_eth_rx_burst(internals->current_primary_port, 124 bd_rx_q->queue_id, bufs, nb_pkts); 125 } 126 127 static inline uint8_t 128 is_lacp_packets(uint16_t ethertype, uint8_t subtype, struct rte_mbuf *mbuf) 129 { 130 const uint16_t ether_type_slow_be = rte_be_to_cpu_16(ETHER_TYPE_SLOW); 131 132 return !((mbuf->ol_flags & PKT_RX_VLAN) ? mbuf->vlan_tci : 0) && 133 (ethertype == ether_type_slow_be && 134 (subtype == SLOW_SUBTYPE_MARKER || subtype == SLOW_SUBTYPE_LACP)); 135 } 136 137 /***************************************************************************** 138 * Flow director's setup for mode 4 optimization 139 */ 140 141 static struct rte_flow_item_eth flow_item_eth_type_8023ad = { 142 .dst.addr_bytes = { 0 }, 143 .src.addr_bytes = { 0 }, 144 .type = RTE_BE16(ETHER_TYPE_SLOW), 145 }; 146 147 static struct rte_flow_item_eth flow_item_eth_mask_type_8023ad = { 148 .dst.addr_bytes = { 0 }, 149 .src.addr_bytes = { 0 }, 150 .type = 0xFFFF, 151 }; 152 153 static struct rte_flow_item flow_item_8023ad[] = { 154 { 155 .type = RTE_FLOW_ITEM_TYPE_ETH, 156 .spec = &flow_item_eth_type_8023ad, 157 .last = NULL, 158 .mask = &flow_item_eth_mask_type_8023ad, 159 }, 160 { 161 .type = RTE_FLOW_ITEM_TYPE_END, 162 .spec = NULL, 163 .last = NULL, 164 .mask = NULL, 165 } 166 }; 167 168 const struct rte_flow_attr flow_attr_8023ad = { 169 .group = 0, 170 .priority = 0, 171 .ingress = 1, 172 .egress = 0, 173 .reserved = 0, 174 }; 175 176 int 177 bond_ethdev_8023ad_flow_verify(struct rte_eth_dev *bond_dev, 178 uint16_t slave_port) { 179 struct rte_eth_dev_info slave_info; 180 struct rte_flow_error error; 181 struct bond_dev_private *internals = (struct bond_dev_private *) 182 (bond_dev->data->dev_private); 183 184 const struct rte_flow_action_queue lacp_queue_conf = { 185 .index = 0, 186 }; 187 188 const struct rte_flow_action actions[] = { 189 { 190 .type = RTE_FLOW_ACTION_TYPE_QUEUE, 191 .conf = &lacp_queue_conf 192 }, 193 { 194 .type = RTE_FLOW_ACTION_TYPE_END, 195 } 196 }; 197 198 int ret = rte_flow_validate(slave_port, &flow_attr_8023ad, 199 flow_item_8023ad, actions, &error); 200 if (ret < 0) { 201 RTE_BOND_LOG(ERR, "%s: %s (slave_port=%d queue_id=%d)", 202 __func__, error.message, slave_port, 203 internals->mode4.dedicated_queues.rx_qid); 204 return -1; 205 } 206 207 rte_eth_dev_info_get(slave_port, &slave_info); 208 if (slave_info.max_rx_queues < bond_dev->data->nb_rx_queues || 209 slave_info.max_tx_queues < bond_dev->data->nb_tx_queues) { 210 RTE_BOND_LOG(ERR, 211 "%s: Slave %d capabilities doesn't allow to allocate additional queues", 212 __func__, slave_port); 213 return -1; 214 } 215 216 return 0; 217 } 218 219 int 220 bond_8023ad_slow_pkt_hw_filter_supported(uint16_t port_id) { 221 struct rte_eth_dev *bond_dev = &rte_eth_devices[port_id]; 222 struct bond_dev_private *internals = (struct bond_dev_private *) 223 (bond_dev->data->dev_private); 224 struct rte_eth_dev_info bond_info; 225 uint16_t idx; 226 227 /* Verify if all slaves in bonding supports flow director and */ 228 if (internals->slave_count > 0) { 229 rte_eth_dev_info_get(bond_dev->data->port_id, &bond_info); 230 231 internals->mode4.dedicated_queues.rx_qid = bond_info.nb_rx_queues; 232 internals->mode4.dedicated_queues.tx_qid = bond_info.nb_tx_queues; 233 234 for (idx = 0; idx < internals->slave_count; idx++) { 235 if (bond_ethdev_8023ad_flow_verify(bond_dev, 236 internals->slaves[idx].port_id) != 0) 237 return -1; 238 } 239 } 240 241 return 0; 242 } 243 244 int 245 bond_ethdev_8023ad_flow_set(struct rte_eth_dev *bond_dev, uint16_t slave_port) { 246 247 struct rte_flow_error error; 248 struct bond_dev_private *internals = (struct bond_dev_private *) 249 (bond_dev->data->dev_private); 250 251 struct rte_flow_action_queue lacp_queue_conf = { 252 .index = internals->mode4.dedicated_queues.rx_qid, 253 }; 254 255 const struct rte_flow_action actions[] = { 256 { 257 .type = RTE_FLOW_ACTION_TYPE_QUEUE, 258 .conf = &lacp_queue_conf 259 }, 260 { 261 .type = RTE_FLOW_ACTION_TYPE_END, 262 } 263 }; 264 265 internals->mode4.dedicated_queues.flow[slave_port] = rte_flow_create(slave_port, 266 &flow_attr_8023ad, flow_item_8023ad, actions, &error); 267 if (internals->mode4.dedicated_queues.flow[slave_port] == NULL) { 268 RTE_BOND_LOG(ERR, "bond_ethdev_8023ad_flow_set: %s " 269 "(slave_port=%d queue_id=%d)", 270 error.message, slave_port, 271 internals->mode4.dedicated_queues.rx_qid); 272 return -1; 273 } 274 275 return 0; 276 } 277 278 static uint16_t 279 bond_ethdev_rx_burst_8023ad_fast_queue(void *queue, struct rte_mbuf **bufs, 280 uint16_t nb_pkts) 281 { 282 struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *)queue; 283 struct bond_dev_private *internals = bd_rx_q->dev_private; 284 uint16_t num_rx_total = 0; /* Total number of received packets */ 285 uint16_t slaves[RTE_MAX_ETHPORTS]; 286 uint16_t slave_count; 287 288 uint16_t i, idx; 289 290 /* Copy slave list to protect against slave up/down changes during tx 291 * bursting */ 292 slave_count = internals->active_slave_count; 293 memcpy(slaves, internals->active_slaves, 294 sizeof(internals->active_slaves[0]) * slave_count); 295 296 for (i = 0, idx = internals->active_slave; 297 i < slave_count && num_rx_total < nb_pkts; i++, idx++) { 298 idx = idx % slave_count; 299 300 /* Read packets from this slave */ 301 num_rx_total += rte_eth_rx_burst(slaves[idx], bd_rx_q->queue_id, 302 &bufs[num_rx_total], nb_pkts - num_rx_total); 303 } 304 305 internals->active_slave = idx; 306 307 return num_rx_total; 308 } 309 310 static uint16_t 311 bond_ethdev_tx_burst_8023ad_fast_queue(void *queue, struct rte_mbuf **bufs, 312 uint16_t nb_pkts) 313 { 314 struct bond_dev_private *internals; 315 struct bond_tx_queue *bd_tx_q; 316 317 uint16_t num_of_slaves; 318 uint16_t slaves[RTE_MAX_ETHPORTS]; 319 /* positions in slaves, not ID */ 320 uint8_t distributing_offsets[RTE_MAX_ETHPORTS]; 321 uint8_t distributing_count; 322 323 uint16_t num_tx_slave, num_tx_total = 0, num_tx_fail_total = 0; 324 uint16_t i, op_slave_idx; 325 326 struct rte_mbuf *slave_bufs[RTE_MAX_ETHPORTS][nb_pkts]; 327 328 /* Total amount of packets in slave_bufs */ 329 uint16_t slave_nb_pkts[RTE_MAX_ETHPORTS] = { 0 }; 330 /* Slow packets placed in each slave */ 331 332 if (unlikely(nb_pkts == 0)) 333 return 0; 334 335 bd_tx_q = (struct bond_tx_queue *)queue; 336 internals = bd_tx_q->dev_private; 337 338 /* Copy slave list to protect against slave up/down changes during tx 339 * bursting */ 340 num_of_slaves = internals->active_slave_count; 341 if (num_of_slaves < 1) 342 return num_tx_total; 343 344 memcpy(slaves, internals->active_slaves, sizeof(slaves[0]) * 345 num_of_slaves); 346 347 distributing_count = 0; 348 for (i = 0; i < num_of_slaves; i++) { 349 struct port *port = &mode_8023ad_ports[slaves[i]]; 350 if (ACTOR_STATE(port, DISTRIBUTING)) 351 distributing_offsets[distributing_count++] = i; 352 } 353 354 if (likely(distributing_count > 0)) { 355 /* Populate slaves mbuf with the packets which are to be sent */ 356 for (i = 0; i < nb_pkts; i++) { 357 /* Select output slave using hash based on xmit policy */ 358 op_slave_idx = internals->xmit_hash(bufs[i], 359 distributing_count); 360 361 /* Populate slave mbuf arrays with mbufs for that slave. 362 * Use only slaves that are currently distributing. 363 */ 364 uint8_t slave_offset = 365 distributing_offsets[op_slave_idx]; 366 slave_bufs[slave_offset][slave_nb_pkts[slave_offset]] = 367 bufs[i]; 368 slave_nb_pkts[slave_offset]++; 369 } 370 } 371 372 /* Send packet burst on each slave device */ 373 for (i = 0; i < num_of_slaves; i++) { 374 if (slave_nb_pkts[i] == 0) 375 continue; 376 377 num_tx_slave = rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id, 378 slave_bufs[i], slave_nb_pkts[i]); 379 380 num_tx_total += num_tx_slave; 381 num_tx_fail_total += slave_nb_pkts[i] - num_tx_slave; 382 383 /* If tx burst fails move packets to end of bufs */ 384 if (unlikely(num_tx_slave < slave_nb_pkts[i])) { 385 uint16_t j = nb_pkts - num_tx_fail_total; 386 for ( ; num_tx_slave < slave_nb_pkts[i]; j++, 387 num_tx_slave++) 388 bufs[j] = slave_bufs[i][num_tx_slave]; 389 } 390 } 391 392 return num_tx_total; 393 } 394 395 396 static uint16_t 397 bond_ethdev_rx_burst_8023ad(void *queue, struct rte_mbuf **bufs, 398 uint16_t nb_pkts) 399 { 400 /* Cast to structure, containing bonded device's port id and queue id */ 401 struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *)queue; 402 struct bond_dev_private *internals = bd_rx_q->dev_private; 403 struct ether_addr bond_mac; 404 405 struct ether_hdr *hdr; 406 407 const uint16_t ether_type_slow_be = rte_be_to_cpu_16(ETHER_TYPE_SLOW); 408 uint16_t num_rx_total = 0; /* Total number of received packets */ 409 uint16_t slaves[RTE_MAX_ETHPORTS]; 410 uint16_t slave_count, idx; 411 412 uint8_t collecting; /* current slave collecting status */ 413 const uint8_t promisc = internals->promiscuous_en; 414 uint8_t i, j, k; 415 uint8_t subtype; 416 417 rte_eth_macaddr_get(internals->port_id, &bond_mac); 418 /* Copy slave list to protect against slave up/down changes during tx 419 * bursting */ 420 slave_count = internals->active_slave_count; 421 memcpy(slaves, internals->active_slaves, 422 sizeof(internals->active_slaves[0]) * slave_count); 423 424 idx = internals->active_slave; 425 if (idx >= slave_count) { 426 internals->active_slave = 0; 427 idx = 0; 428 } 429 for (i = 0; i < slave_count && num_rx_total < nb_pkts; i++) { 430 j = num_rx_total; 431 collecting = ACTOR_STATE(&mode_8023ad_ports[slaves[idx]], 432 COLLECTING); 433 434 /* Read packets from this slave */ 435 num_rx_total += rte_eth_rx_burst(slaves[idx], bd_rx_q->queue_id, 436 &bufs[num_rx_total], nb_pkts - num_rx_total); 437 438 for (k = j; k < 2 && k < num_rx_total; k++) 439 rte_prefetch0(rte_pktmbuf_mtod(bufs[k], void *)); 440 441 /* Handle slow protocol packets. */ 442 while (j < num_rx_total) { 443 444 /* If packet is not pure L2 and is known, skip it */ 445 if ((bufs[j]->packet_type & ~RTE_PTYPE_L2_ETHER) != 0) { 446 j++; 447 continue; 448 } 449 450 if (j + 3 < num_rx_total) 451 rte_prefetch0(rte_pktmbuf_mtod(bufs[j + 3], void *)); 452 453 hdr = rte_pktmbuf_mtod(bufs[j], struct ether_hdr *); 454 subtype = ((struct slow_protocol_frame *)hdr)->slow_protocol.subtype; 455 456 /* Remove packet from array if it is slow packet or slave is not 457 * in collecting state or bonding interface is not in promiscuous 458 * mode and packet address does not match. */ 459 if (unlikely(is_lacp_packets(hdr->ether_type, subtype, bufs[j]) || 460 !collecting || (!promisc && 461 !is_multicast_ether_addr(&hdr->d_addr) && 462 !is_same_ether_addr(&bond_mac, &hdr->d_addr)))) { 463 464 if (hdr->ether_type == ether_type_slow_be) { 465 bond_mode_8023ad_handle_slow_pkt( 466 internals, slaves[idx], bufs[j]); 467 } else 468 rte_pktmbuf_free(bufs[j]); 469 470 /* Packet is managed by mode 4 or dropped, shift the array */ 471 num_rx_total--; 472 if (j < num_rx_total) { 473 memmove(&bufs[j], &bufs[j + 1], sizeof(bufs[0]) * 474 (num_rx_total - j)); 475 } 476 } else 477 j++; 478 } 479 if (unlikely(++idx == slave_count)) 480 idx = 0; 481 } 482 483 internals->active_slave = idx; 484 return num_rx_total; 485 } 486 487 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1) 488 uint32_t burstnumberRX; 489 uint32_t burstnumberTX; 490 491 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB 492 493 static void 494 arp_op_name(uint16_t arp_op, char *buf) 495 { 496 switch (arp_op) { 497 case ARP_OP_REQUEST: 498 snprintf(buf, sizeof("ARP Request"), "%s", "ARP Request"); 499 return; 500 case ARP_OP_REPLY: 501 snprintf(buf, sizeof("ARP Reply"), "%s", "ARP Reply"); 502 return; 503 case ARP_OP_REVREQUEST: 504 snprintf(buf, sizeof("Reverse ARP Request"), "%s", 505 "Reverse ARP Request"); 506 return; 507 case ARP_OP_REVREPLY: 508 snprintf(buf, sizeof("Reverse ARP Reply"), "%s", 509 "Reverse ARP Reply"); 510 return; 511 case ARP_OP_INVREQUEST: 512 snprintf(buf, sizeof("Peer Identify Request"), "%s", 513 "Peer Identify Request"); 514 return; 515 case ARP_OP_INVREPLY: 516 snprintf(buf, sizeof("Peer Identify Reply"), "%s", 517 "Peer Identify Reply"); 518 return; 519 default: 520 break; 521 } 522 snprintf(buf, sizeof("Unknown"), "%s", "Unknown"); 523 return; 524 } 525 #endif 526 #define MaxIPv4String 16 527 static void 528 ipv4_addr_to_dot(uint32_t be_ipv4_addr, char *buf, uint8_t buf_size) 529 { 530 uint32_t ipv4_addr; 531 532 ipv4_addr = rte_be_to_cpu_32(be_ipv4_addr); 533 snprintf(buf, buf_size, "%d.%d.%d.%d", (ipv4_addr >> 24) & 0xFF, 534 (ipv4_addr >> 16) & 0xFF, (ipv4_addr >> 8) & 0xFF, 535 ipv4_addr & 0xFF); 536 } 537 538 #define MAX_CLIENTS_NUMBER 128 539 uint8_t active_clients; 540 struct client_stats_t { 541 uint16_t port; 542 uint32_t ipv4_addr; 543 uint32_t ipv4_rx_packets; 544 uint32_t ipv4_tx_packets; 545 }; 546 struct client_stats_t client_stats[MAX_CLIENTS_NUMBER]; 547 548 static void 549 update_client_stats(uint32_t addr, uint16_t port, uint32_t *TXorRXindicator) 550 { 551 int i = 0; 552 553 for (; i < MAX_CLIENTS_NUMBER; i++) { 554 if ((client_stats[i].ipv4_addr == addr) && (client_stats[i].port == port)) { 555 /* Just update RX packets number for this client */ 556 if (TXorRXindicator == &burstnumberRX) 557 client_stats[i].ipv4_rx_packets++; 558 else 559 client_stats[i].ipv4_tx_packets++; 560 return; 561 } 562 } 563 /* We have a new client. Insert him to the table, and increment stats */ 564 if (TXorRXindicator == &burstnumberRX) 565 client_stats[active_clients].ipv4_rx_packets++; 566 else 567 client_stats[active_clients].ipv4_tx_packets++; 568 client_stats[active_clients].ipv4_addr = addr; 569 client_stats[active_clients].port = port; 570 active_clients++; 571 572 } 573 574 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB 575 #define MODE6_DEBUG(info, src_ip, dst_ip, eth_h, arp_op, port, burstnumber) \ 576 RTE_LOG(DEBUG, PMD, \ 577 "%s " \ 578 "port:%d " \ 579 "SrcMAC:%02X:%02X:%02X:%02X:%02X:%02X " \ 580 "SrcIP:%s " \ 581 "DstMAC:%02X:%02X:%02X:%02X:%02X:%02X " \ 582 "DstIP:%s " \ 583 "%s " \ 584 "%d\n", \ 585 info, \ 586 port, \ 587 eth_h->s_addr.addr_bytes[0], \ 588 eth_h->s_addr.addr_bytes[1], \ 589 eth_h->s_addr.addr_bytes[2], \ 590 eth_h->s_addr.addr_bytes[3], \ 591 eth_h->s_addr.addr_bytes[4], \ 592 eth_h->s_addr.addr_bytes[5], \ 593 src_ip, \ 594 eth_h->d_addr.addr_bytes[0], \ 595 eth_h->d_addr.addr_bytes[1], \ 596 eth_h->d_addr.addr_bytes[2], \ 597 eth_h->d_addr.addr_bytes[3], \ 598 eth_h->d_addr.addr_bytes[4], \ 599 eth_h->d_addr.addr_bytes[5], \ 600 dst_ip, \ 601 arp_op, \ 602 ++burstnumber) 603 #endif 604 605 static void 606 mode6_debug(const char __attribute__((unused)) *info, struct ether_hdr *eth_h, 607 uint16_t port, uint32_t __attribute__((unused)) *burstnumber) 608 { 609 struct ipv4_hdr *ipv4_h; 610 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB 611 struct arp_hdr *arp_h; 612 char dst_ip[16]; 613 char ArpOp[24]; 614 char buf[16]; 615 #endif 616 char src_ip[16]; 617 618 uint16_t ether_type = eth_h->ether_type; 619 uint16_t offset = get_vlan_offset(eth_h, ðer_type); 620 621 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB 622 snprintf(buf, 16, "%s", info); 623 #endif 624 625 if (ether_type == rte_cpu_to_be_16(ETHER_TYPE_IPv4)) { 626 ipv4_h = (struct ipv4_hdr *)((char *)(eth_h + 1) + offset); 627 ipv4_addr_to_dot(ipv4_h->src_addr, src_ip, MaxIPv4String); 628 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB 629 ipv4_addr_to_dot(ipv4_h->dst_addr, dst_ip, MaxIPv4String); 630 MODE6_DEBUG(buf, src_ip, dst_ip, eth_h, "", port, *burstnumber); 631 #endif 632 update_client_stats(ipv4_h->src_addr, port, burstnumber); 633 } 634 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB 635 else if (ether_type == rte_cpu_to_be_16(ETHER_TYPE_ARP)) { 636 arp_h = (struct arp_hdr *)((char *)(eth_h + 1) + offset); 637 ipv4_addr_to_dot(arp_h->arp_data.arp_sip, src_ip, MaxIPv4String); 638 ipv4_addr_to_dot(arp_h->arp_data.arp_tip, dst_ip, MaxIPv4String); 639 arp_op_name(rte_be_to_cpu_16(arp_h->arp_op), ArpOp); 640 MODE6_DEBUG(buf, src_ip, dst_ip, eth_h, ArpOp, port, *burstnumber); 641 } 642 #endif 643 } 644 #endif 645 646 static uint16_t 647 bond_ethdev_rx_burst_alb(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) 648 { 649 struct bond_tx_queue *bd_tx_q = (struct bond_tx_queue *)queue; 650 struct bond_dev_private *internals = bd_tx_q->dev_private; 651 struct ether_hdr *eth_h; 652 uint16_t ether_type, offset; 653 uint16_t nb_recv_pkts; 654 int i; 655 656 nb_recv_pkts = bond_ethdev_rx_burst(queue, bufs, nb_pkts); 657 658 for (i = 0; i < nb_recv_pkts; i++) { 659 eth_h = rte_pktmbuf_mtod(bufs[i], struct ether_hdr *); 660 ether_type = eth_h->ether_type; 661 offset = get_vlan_offset(eth_h, ðer_type); 662 663 if (ether_type == rte_cpu_to_be_16(ETHER_TYPE_ARP)) { 664 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1) 665 mode6_debug("RX ARP:", eth_h, bufs[i]->port, &burstnumberRX); 666 #endif 667 bond_mode_alb_arp_recv(eth_h, offset, internals); 668 } 669 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1) 670 else if (ether_type == rte_cpu_to_be_16(ETHER_TYPE_IPv4)) 671 mode6_debug("RX IPv4:", eth_h, bufs[i]->port, &burstnumberRX); 672 #endif 673 } 674 675 return nb_recv_pkts; 676 } 677 678 static uint16_t 679 bond_ethdev_tx_burst_round_robin(void *queue, struct rte_mbuf **bufs, 680 uint16_t nb_pkts) 681 { 682 struct bond_dev_private *internals; 683 struct bond_tx_queue *bd_tx_q; 684 685 struct rte_mbuf *slave_bufs[RTE_MAX_ETHPORTS][nb_pkts]; 686 uint16_t slave_nb_pkts[RTE_MAX_ETHPORTS] = { 0 }; 687 688 uint16_t num_of_slaves; 689 uint16_t slaves[RTE_MAX_ETHPORTS]; 690 691 uint16_t num_tx_total = 0, num_tx_slave; 692 693 static int slave_idx = 0; 694 int i, cslave_idx = 0, tx_fail_total = 0; 695 696 bd_tx_q = (struct bond_tx_queue *)queue; 697 internals = bd_tx_q->dev_private; 698 699 /* Copy slave list to protect against slave up/down changes during tx 700 * bursting */ 701 num_of_slaves = internals->active_slave_count; 702 memcpy(slaves, internals->active_slaves, 703 sizeof(internals->active_slaves[0]) * num_of_slaves); 704 705 if (num_of_slaves < 1) 706 return num_tx_total; 707 708 /* Populate slaves mbuf with which packets are to be sent on it */ 709 for (i = 0; i < nb_pkts; i++) { 710 cslave_idx = (slave_idx + i) % num_of_slaves; 711 slave_bufs[cslave_idx][(slave_nb_pkts[cslave_idx])++] = bufs[i]; 712 } 713 714 /* increment current slave index so the next call to tx burst starts on the 715 * next slave */ 716 slave_idx = ++cslave_idx; 717 718 /* Send packet burst on each slave device */ 719 for (i = 0; i < num_of_slaves; i++) { 720 if (slave_nb_pkts[i] > 0) { 721 num_tx_slave = rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id, 722 slave_bufs[i], slave_nb_pkts[i]); 723 724 /* if tx burst fails move packets to end of bufs */ 725 if (unlikely(num_tx_slave < slave_nb_pkts[i])) { 726 int tx_fail_slave = slave_nb_pkts[i] - num_tx_slave; 727 728 tx_fail_total += tx_fail_slave; 729 730 memcpy(&bufs[nb_pkts - tx_fail_total], 731 &slave_bufs[i][num_tx_slave], 732 tx_fail_slave * sizeof(bufs[0])); 733 } 734 num_tx_total += num_tx_slave; 735 } 736 } 737 738 return num_tx_total; 739 } 740 741 static uint16_t 742 bond_ethdev_tx_burst_active_backup(void *queue, 743 struct rte_mbuf **bufs, uint16_t nb_pkts) 744 { 745 struct bond_dev_private *internals; 746 struct bond_tx_queue *bd_tx_q; 747 748 bd_tx_q = (struct bond_tx_queue *)queue; 749 internals = bd_tx_q->dev_private; 750 751 if (internals->active_slave_count < 1) 752 return 0; 753 754 return rte_eth_tx_burst(internals->current_primary_port, bd_tx_q->queue_id, 755 bufs, nb_pkts); 756 } 757 758 static inline uint16_t 759 ether_hash(struct ether_hdr *eth_hdr) 760 { 761 unaligned_uint16_t *word_src_addr = 762 (unaligned_uint16_t *)eth_hdr->s_addr.addr_bytes; 763 unaligned_uint16_t *word_dst_addr = 764 (unaligned_uint16_t *)eth_hdr->d_addr.addr_bytes; 765 766 return (word_src_addr[0] ^ word_dst_addr[0]) ^ 767 (word_src_addr[1] ^ word_dst_addr[1]) ^ 768 (word_src_addr[2] ^ word_dst_addr[2]); 769 } 770 771 static inline uint32_t 772 ipv4_hash(struct ipv4_hdr *ipv4_hdr) 773 { 774 return ipv4_hdr->src_addr ^ ipv4_hdr->dst_addr; 775 } 776 777 static inline uint32_t 778 ipv6_hash(struct ipv6_hdr *ipv6_hdr) 779 { 780 unaligned_uint32_t *word_src_addr = 781 (unaligned_uint32_t *)&(ipv6_hdr->src_addr[0]); 782 unaligned_uint32_t *word_dst_addr = 783 (unaligned_uint32_t *)&(ipv6_hdr->dst_addr[0]); 784 785 return (word_src_addr[0] ^ word_dst_addr[0]) ^ 786 (word_src_addr[1] ^ word_dst_addr[1]) ^ 787 (word_src_addr[2] ^ word_dst_addr[2]) ^ 788 (word_src_addr[3] ^ word_dst_addr[3]); 789 } 790 791 uint16_t 792 xmit_l2_hash(const struct rte_mbuf *buf, uint8_t slave_count) 793 { 794 struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(buf, struct ether_hdr *); 795 796 uint32_t hash = ether_hash(eth_hdr); 797 798 return (hash ^= hash >> 8) % slave_count; 799 } 800 801 uint16_t 802 xmit_l23_hash(const struct rte_mbuf *buf, uint8_t slave_count) 803 { 804 struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(buf, struct ether_hdr *); 805 uint16_t proto = eth_hdr->ether_type; 806 size_t vlan_offset = get_vlan_offset(eth_hdr, &proto); 807 uint32_t hash, l3hash = 0; 808 809 hash = ether_hash(eth_hdr); 810 811 if (rte_cpu_to_be_16(ETHER_TYPE_IPv4) == proto) { 812 struct ipv4_hdr *ipv4_hdr = (struct ipv4_hdr *) 813 ((char *)(eth_hdr + 1) + vlan_offset); 814 l3hash = ipv4_hash(ipv4_hdr); 815 816 } else if (rte_cpu_to_be_16(ETHER_TYPE_IPv6) == proto) { 817 struct ipv6_hdr *ipv6_hdr = (struct ipv6_hdr *) 818 ((char *)(eth_hdr + 1) + vlan_offset); 819 l3hash = ipv6_hash(ipv6_hdr); 820 } 821 822 hash = hash ^ l3hash; 823 hash ^= hash >> 16; 824 hash ^= hash >> 8; 825 826 return hash % slave_count; 827 } 828 829 uint16_t 830 xmit_l34_hash(const struct rte_mbuf *buf, uint8_t slave_count) 831 { 832 struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(buf, struct ether_hdr *); 833 uint16_t proto = eth_hdr->ether_type; 834 size_t vlan_offset = get_vlan_offset(eth_hdr, &proto); 835 836 struct udp_hdr *udp_hdr = NULL; 837 struct tcp_hdr *tcp_hdr = NULL; 838 uint32_t hash, l3hash = 0, l4hash = 0; 839 840 if (rte_cpu_to_be_16(ETHER_TYPE_IPv4) == proto) { 841 struct ipv4_hdr *ipv4_hdr = (struct ipv4_hdr *) 842 ((char *)(eth_hdr + 1) + vlan_offset); 843 size_t ip_hdr_offset; 844 845 l3hash = ipv4_hash(ipv4_hdr); 846 847 /* there is no L4 header in fragmented packet */ 848 if (likely(rte_ipv4_frag_pkt_is_fragmented(ipv4_hdr) == 0)) { 849 ip_hdr_offset = (ipv4_hdr->version_ihl & IPV4_HDR_IHL_MASK) * 850 IPV4_IHL_MULTIPLIER; 851 852 if (ipv4_hdr->next_proto_id == IPPROTO_TCP) { 853 tcp_hdr = (struct tcp_hdr *)((char *)ipv4_hdr + 854 ip_hdr_offset); 855 l4hash = HASH_L4_PORTS(tcp_hdr); 856 } else if (ipv4_hdr->next_proto_id == IPPROTO_UDP) { 857 udp_hdr = (struct udp_hdr *)((char *)ipv4_hdr + 858 ip_hdr_offset); 859 l4hash = HASH_L4_PORTS(udp_hdr); 860 } 861 } 862 } else if (rte_cpu_to_be_16(ETHER_TYPE_IPv6) == proto) { 863 struct ipv6_hdr *ipv6_hdr = (struct ipv6_hdr *) 864 ((char *)(eth_hdr + 1) + vlan_offset); 865 l3hash = ipv6_hash(ipv6_hdr); 866 867 if (ipv6_hdr->proto == IPPROTO_TCP) { 868 tcp_hdr = (struct tcp_hdr *)(ipv6_hdr + 1); 869 l4hash = HASH_L4_PORTS(tcp_hdr); 870 } else if (ipv6_hdr->proto == IPPROTO_UDP) { 871 udp_hdr = (struct udp_hdr *)(ipv6_hdr + 1); 872 l4hash = HASH_L4_PORTS(udp_hdr); 873 } 874 } 875 876 hash = l3hash ^ l4hash; 877 hash ^= hash >> 16; 878 hash ^= hash >> 8; 879 880 return hash % slave_count; 881 } 882 883 struct bwg_slave { 884 uint64_t bwg_left_int; 885 uint64_t bwg_left_remainder; 886 uint8_t slave; 887 }; 888 889 void 890 bond_tlb_activate_slave(struct bond_dev_private *internals) { 891 int i; 892 893 for (i = 0; i < internals->active_slave_count; i++) { 894 tlb_last_obytets[internals->active_slaves[i]] = 0; 895 } 896 } 897 898 static int 899 bandwidth_cmp(const void *a, const void *b) 900 { 901 const struct bwg_slave *bwg_a = a; 902 const struct bwg_slave *bwg_b = b; 903 int64_t diff = (int64_t)bwg_b->bwg_left_int - (int64_t)bwg_a->bwg_left_int; 904 int64_t diff2 = (int64_t)bwg_b->bwg_left_remainder - 905 (int64_t)bwg_a->bwg_left_remainder; 906 if (diff > 0) 907 return 1; 908 else if (diff < 0) 909 return -1; 910 else if (diff2 > 0) 911 return 1; 912 else if (diff2 < 0) 913 return -1; 914 else 915 return 0; 916 } 917 918 static void 919 bandwidth_left(uint16_t port_id, uint64_t load, uint8_t update_idx, 920 struct bwg_slave *bwg_slave) 921 { 922 struct rte_eth_link link_status; 923 924 rte_eth_link_get_nowait(port_id, &link_status); 925 uint64_t link_bwg = link_status.link_speed * 1000000ULL / 8; 926 if (link_bwg == 0) 927 return; 928 link_bwg = link_bwg * (update_idx+1) * REORDER_PERIOD_MS; 929 bwg_slave->bwg_left_int = (link_bwg - 1000*load) / link_bwg; 930 bwg_slave->bwg_left_remainder = (link_bwg - 1000*load) % link_bwg; 931 } 932 933 static void 934 bond_ethdev_update_tlb_slave_cb(void *arg) 935 { 936 struct bond_dev_private *internals = arg; 937 struct rte_eth_stats slave_stats; 938 struct bwg_slave bwg_array[RTE_MAX_ETHPORTS]; 939 uint8_t slave_count; 940 uint64_t tx_bytes; 941 942 uint8_t update_stats = 0; 943 uint8_t i, slave_id; 944 945 internals->slave_update_idx++; 946 947 948 if (internals->slave_update_idx >= REORDER_PERIOD_MS) 949 update_stats = 1; 950 951 for (i = 0; i < internals->active_slave_count; i++) { 952 slave_id = internals->active_slaves[i]; 953 rte_eth_stats_get(slave_id, &slave_stats); 954 tx_bytes = slave_stats.obytes - tlb_last_obytets[slave_id]; 955 bandwidth_left(slave_id, tx_bytes, 956 internals->slave_update_idx, &bwg_array[i]); 957 bwg_array[i].slave = slave_id; 958 959 if (update_stats) { 960 tlb_last_obytets[slave_id] = slave_stats.obytes; 961 } 962 } 963 964 if (update_stats == 1) 965 internals->slave_update_idx = 0; 966 967 slave_count = i; 968 qsort(bwg_array, slave_count, sizeof(bwg_array[0]), bandwidth_cmp); 969 for (i = 0; i < slave_count; i++) 970 internals->tlb_slaves_order[i] = bwg_array[i].slave; 971 972 rte_eal_alarm_set(REORDER_PERIOD_MS * 1000, bond_ethdev_update_tlb_slave_cb, 973 (struct bond_dev_private *)internals); 974 } 975 976 static uint16_t 977 bond_ethdev_tx_burst_tlb(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) 978 { 979 struct bond_tx_queue *bd_tx_q = (struct bond_tx_queue *)queue; 980 struct bond_dev_private *internals = bd_tx_q->dev_private; 981 982 struct rte_eth_dev *primary_port = 983 &rte_eth_devices[internals->primary_port]; 984 uint16_t num_tx_total = 0; 985 uint16_t i, j; 986 987 uint16_t num_of_slaves = internals->active_slave_count; 988 uint16_t slaves[RTE_MAX_ETHPORTS]; 989 990 struct ether_hdr *ether_hdr; 991 struct ether_addr primary_slave_addr; 992 struct ether_addr active_slave_addr; 993 994 if (num_of_slaves < 1) 995 return num_tx_total; 996 997 memcpy(slaves, internals->tlb_slaves_order, 998 sizeof(internals->tlb_slaves_order[0]) * num_of_slaves); 999 1000 1001 ether_addr_copy(primary_port->data->mac_addrs, &primary_slave_addr); 1002 1003 if (nb_pkts > 3) { 1004 for (i = 0; i < 3; i++) 1005 rte_prefetch0(rte_pktmbuf_mtod(bufs[i], void*)); 1006 } 1007 1008 for (i = 0; i < num_of_slaves; i++) { 1009 rte_eth_macaddr_get(slaves[i], &active_slave_addr); 1010 for (j = num_tx_total; j < nb_pkts; j++) { 1011 if (j + 3 < nb_pkts) 1012 rte_prefetch0(rte_pktmbuf_mtod(bufs[j+3], void*)); 1013 1014 ether_hdr = rte_pktmbuf_mtod(bufs[j], struct ether_hdr *); 1015 if (is_same_ether_addr(ðer_hdr->s_addr, &primary_slave_addr)) 1016 ether_addr_copy(&active_slave_addr, ðer_hdr->s_addr); 1017 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1) 1018 mode6_debug("TX IPv4:", ether_hdr, slaves[i], &burstnumberTX); 1019 #endif 1020 } 1021 1022 num_tx_total += rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id, 1023 bufs + num_tx_total, nb_pkts - num_tx_total); 1024 1025 if (num_tx_total == nb_pkts) 1026 break; 1027 } 1028 1029 return num_tx_total; 1030 } 1031 1032 void 1033 bond_tlb_disable(struct bond_dev_private *internals) 1034 { 1035 rte_eal_alarm_cancel(bond_ethdev_update_tlb_slave_cb, internals); 1036 } 1037 1038 void 1039 bond_tlb_enable(struct bond_dev_private *internals) 1040 { 1041 bond_ethdev_update_tlb_slave_cb(internals); 1042 } 1043 1044 static uint16_t 1045 bond_ethdev_tx_burst_alb(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) 1046 { 1047 struct bond_tx_queue *bd_tx_q = (struct bond_tx_queue *)queue; 1048 struct bond_dev_private *internals = bd_tx_q->dev_private; 1049 1050 struct ether_hdr *eth_h; 1051 uint16_t ether_type, offset; 1052 1053 struct client_data *client_info; 1054 1055 /* 1056 * We create transmit buffers for every slave and one additional to send 1057 * through tlb. In worst case every packet will be send on one port. 1058 */ 1059 struct rte_mbuf *slave_bufs[RTE_MAX_ETHPORTS + 1][nb_pkts]; 1060 uint16_t slave_bufs_pkts[RTE_MAX_ETHPORTS + 1] = { 0 }; 1061 1062 /* 1063 * We create separate transmit buffers for update packets as they won't 1064 * be counted in num_tx_total. 1065 */ 1066 struct rte_mbuf *update_bufs[RTE_MAX_ETHPORTS][ALB_HASH_TABLE_SIZE]; 1067 uint16_t update_bufs_pkts[RTE_MAX_ETHPORTS] = { 0 }; 1068 1069 struct rte_mbuf *upd_pkt; 1070 size_t pkt_size; 1071 1072 uint16_t num_send, num_not_send = 0; 1073 uint16_t num_tx_total = 0; 1074 uint16_t slave_idx; 1075 1076 int i, j; 1077 1078 /* Search tx buffer for ARP packets and forward them to alb */ 1079 for (i = 0; i < nb_pkts; i++) { 1080 eth_h = rte_pktmbuf_mtod(bufs[i], struct ether_hdr *); 1081 ether_type = eth_h->ether_type; 1082 offset = get_vlan_offset(eth_h, ðer_type); 1083 1084 if (ether_type == rte_cpu_to_be_16(ETHER_TYPE_ARP)) { 1085 slave_idx = bond_mode_alb_arp_xmit(eth_h, offset, internals); 1086 1087 /* Change src mac in eth header */ 1088 rte_eth_macaddr_get(slave_idx, ð_h->s_addr); 1089 1090 /* Add packet to slave tx buffer */ 1091 slave_bufs[slave_idx][slave_bufs_pkts[slave_idx]] = bufs[i]; 1092 slave_bufs_pkts[slave_idx]++; 1093 } else { 1094 /* If packet is not ARP, send it with TLB policy */ 1095 slave_bufs[RTE_MAX_ETHPORTS][slave_bufs_pkts[RTE_MAX_ETHPORTS]] = 1096 bufs[i]; 1097 slave_bufs_pkts[RTE_MAX_ETHPORTS]++; 1098 } 1099 } 1100 1101 /* Update connected client ARP tables */ 1102 if (internals->mode6.ntt) { 1103 for (i = 0; i < ALB_HASH_TABLE_SIZE; i++) { 1104 client_info = &internals->mode6.client_table[i]; 1105 1106 if (client_info->in_use) { 1107 /* Allocate new packet to send ARP update on current slave */ 1108 upd_pkt = rte_pktmbuf_alloc(internals->mode6.mempool); 1109 if (upd_pkt == NULL) { 1110 RTE_LOG(ERR, PMD, "Failed to allocate ARP packet from pool\n"); 1111 continue; 1112 } 1113 pkt_size = sizeof(struct ether_hdr) + sizeof(struct arp_hdr) 1114 + client_info->vlan_count * sizeof(struct vlan_hdr); 1115 upd_pkt->data_len = pkt_size; 1116 upd_pkt->pkt_len = pkt_size; 1117 1118 slave_idx = bond_mode_alb_arp_upd(client_info, upd_pkt, 1119 internals); 1120 1121 /* Add packet to update tx buffer */ 1122 update_bufs[slave_idx][update_bufs_pkts[slave_idx]] = upd_pkt; 1123 update_bufs_pkts[slave_idx]++; 1124 } 1125 } 1126 internals->mode6.ntt = 0; 1127 } 1128 1129 /* Send ARP packets on proper slaves */ 1130 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 1131 if (slave_bufs_pkts[i] > 0) { 1132 num_send = rte_eth_tx_burst(i, bd_tx_q->queue_id, 1133 slave_bufs[i], slave_bufs_pkts[i]); 1134 for (j = 0; j < slave_bufs_pkts[i] - num_send; j++) { 1135 bufs[nb_pkts - 1 - num_not_send - j] = 1136 slave_bufs[i][nb_pkts - 1 - j]; 1137 } 1138 1139 num_tx_total += num_send; 1140 num_not_send += slave_bufs_pkts[i] - num_send; 1141 1142 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1) 1143 /* Print TX stats including update packets */ 1144 for (j = 0; j < slave_bufs_pkts[i]; j++) { 1145 eth_h = rte_pktmbuf_mtod(slave_bufs[i][j], struct ether_hdr *); 1146 mode6_debug("TX ARP:", eth_h, i, &burstnumberTX); 1147 } 1148 #endif 1149 } 1150 } 1151 1152 /* Send update packets on proper slaves */ 1153 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 1154 if (update_bufs_pkts[i] > 0) { 1155 num_send = rte_eth_tx_burst(i, bd_tx_q->queue_id, update_bufs[i], 1156 update_bufs_pkts[i]); 1157 for (j = num_send; j < update_bufs_pkts[i]; j++) { 1158 rte_pktmbuf_free(update_bufs[i][j]); 1159 } 1160 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1) 1161 for (j = 0; j < update_bufs_pkts[i]; j++) { 1162 eth_h = rte_pktmbuf_mtod(update_bufs[i][j], struct ether_hdr *); 1163 mode6_debug("TX ARPupd:", eth_h, i, &burstnumberTX); 1164 } 1165 #endif 1166 } 1167 } 1168 1169 /* Send non-ARP packets using tlb policy */ 1170 if (slave_bufs_pkts[RTE_MAX_ETHPORTS] > 0) { 1171 num_send = bond_ethdev_tx_burst_tlb(queue, 1172 slave_bufs[RTE_MAX_ETHPORTS], 1173 slave_bufs_pkts[RTE_MAX_ETHPORTS]); 1174 1175 for (j = 0; j < slave_bufs_pkts[RTE_MAX_ETHPORTS]; j++) { 1176 bufs[nb_pkts - 1 - num_not_send - j] = 1177 slave_bufs[RTE_MAX_ETHPORTS][nb_pkts - 1 - j]; 1178 } 1179 1180 num_tx_total += num_send; 1181 } 1182 1183 return num_tx_total; 1184 } 1185 1186 static uint16_t 1187 bond_ethdev_tx_burst_balance(void *queue, struct rte_mbuf **bufs, 1188 uint16_t nb_pkts) 1189 { 1190 struct bond_dev_private *internals; 1191 struct bond_tx_queue *bd_tx_q; 1192 1193 uint16_t num_of_slaves; 1194 uint16_t slaves[RTE_MAX_ETHPORTS]; 1195 1196 uint16_t num_tx_total = 0, num_tx_slave = 0, tx_fail_total = 0; 1197 1198 int i, op_slave_id; 1199 1200 struct rte_mbuf *slave_bufs[RTE_MAX_ETHPORTS][nb_pkts]; 1201 uint16_t slave_nb_pkts[RTE_MAX_ETHPORTS] = { 0 }; 1202 1203 bd_tx_q = (struct bond_tx_queue *)queue; 1204 internals = bd_tx_q->dev_private; 1205 1206 /* Copy slave list to protect against slave up/down changes during tx 1207 * bursting */ 1208 num_of_slaves = internals->active_slave_count; 1209 memcpy(slaves, internals->active_slaves, 1210 sizeof(internals->active_slaves[0]) * num_of_slaves); 1211 1212 if (num_of_slaves < 1) 1213 return num_tx_total; 1214 1215 /* Populate slaves mbuf with the packets which are to be sent on it */ 1216 for (i = 0; i < nb_pkts; i++) { 1217 /* Select output slave using hash based on xmit policy */ 1218 op_slave_id = internals->xmit_hash(bufs[i], num_of_slaves); 1219 1220 /* Populate slave mbuf arrays with mbufs for that slave */ 1221 slave_bufs[op_slave_id][slave_nb_pkts[op_slave_id]++] = bufs[i]; 1222 } 1223 1224 /* Send packet burst on each slave device */ 1225 for (i = 0; i < num_of_slaves; i++) { 1226 if (slave_nb_pkts[i] > 0) { 1227 num_tx_slave = rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id, 1228 slave_bufs[i], slave_nb_pkts[i]); 1229 1230 /* if tx burst fails move packets to end of bufs */ 1231 if (unlikely(num_tx_slave < slave_nb_pkts[i])) { 1232 int slave_tx_fail_count = slave_nb_pkts[i] - num_tx_slave; 1233 1234 tx_fail_total += slave_tx_fail_count; 1235 memcpy(&bufs[nb_pkts - tx_fail_total], 1236 &slave_bufs[i][num_tx_slave], 1237 slave_tx_fail_count * sizeof(bufs[0])); 1238 } 1239 1240 num_tx_total += num_tx_slave; 1241 } 1242 } 1243 1244 return num_tx_total; 1245 } 1246 1247 static uint16_t 1248 bond_ethdev_tx_burst_8023ad(void *queue, struct rte_mbuf **bufs, 1249 uint16_t nb_pkts) 1250 { 1251 struct bond_dev_private *internals; 1252 struct bond_tx_queue *bd_tx_q; 1253 1254 uint16_t num_of_slaves; 1255 uint16_t slaves[RTE_MAX_ETHPORTS]; 1256 /* positions in slaves, not ID */ 1257 uint8_t distributing_offsets[RTE_MAX_ETHPORTS]; 1258 uint8_t distributing_count; 1259 1260 uint16_t num_tx_slave, num_tx_total = 0, num_tx_fail_total = 0; 1261 uint16_t i, j, op_slave_idx; 1262 const uint16_t buffs_size = nb_pkts + BOND_MODE_8023AX_SLAVE_TX_PKTS + 1; 1263 1264 /* Allocate additional packets in case 8023AD mode. */ 1265 struct rte_mbuf *slave_bufs[RTE_MAX_ETHPORTS][buffs_size]; 1266 void *slow_pkts[BOND_MODE_8023AX_SLAVE_TX_PKTS] = { NULL }; 1267 1268 /* Total amount of packets in slave_bufs */ 1269 uint16_t slave_nb_pkts[RTE_MAX_ETHPORTS] = { 0 }; 1270 /* Slow packets placed in each slave */ 1271 uint8_t slave_slow_nb_pkts[RTE_MAX_ETHPORTS] = { 0 }; 1272 1273 bd_tx_q = (struct bond_tx_queue *)queue; 1274 internals = bd_tx_q->dev_private; 1275 1276 /* Copy slave list to protect against slave up/down changes during tx 1277 * bursting */ 1278 num_of_slaves = internals->active_slave_count; 1279 if (num_of_slaves < 1) 1280 return num_tx_total; 1281 1282 memcpy(slaves, internals->active_slaves, sizeof(slaves[0]) * num_of_slaves); 1283 1284 distributing_count = 0; 1285 for (i = 0; i < num_of_slaves; i++) { 1286 struct port *port = &mode_8023ad_ports[slaves[i]]; 1287 1288 slave_slow_nb_pkts[i] = rte_ring_dequeue_burst(port->tx_ring, 1289 slow_pkts, BOND_MODE_8023AX_SLAVE_TX_PKTS, 1290 NULL); 1291 slave_nb_pkts[i] = slave_slow_nb_pkts[i]; 1292 1293 for (j = 0; j < slave_slow_nb_pkts[i]; j++) 1294 slave_bufs[i][j] = slow_pkts[j]; 1295 1296 if (ACTOR_STATE(port, DISTRIBUTING)) 1297 distributing_offsets[distributing_count++] = i; 1298 } 1299 1300 if (likely(distributing_count > 0)) { 1301 /* Populate slaves mbuf with the packets which are to be sent on it */ 1302 for (i = 0; i < nb_pkts; i++) { 1303 /* Select output slave using hash based on xmit policy */ 1304 op_slave_idx = internals->xmit_hash(bufs[i], distributing_count); 1305 1306 /* Populate slave mbuf arrays with mbufs for that slave. Use only 1307 * slaves that are currently distributing. */ 1308 uint8_t slave_offset = distributing_offsets[op_slave_idx]; 1309 slave_bufs[slave_offset][slave_nb_pkts[slave_offset]] = bufs[i]; 1310 slave_nb_pkts[slave_offset]++; 1311 } 1312 } 1313 1314 /* Send packet burst on each slave device */ 1315 for (i = 0; i < num_of_slaves; i++) { 1316 if (slave_nb_pkts[i] == 0) 1317 continue; 1318 1319 num_tx_slave = rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id, 1320 slave_bufs[i], slave_nb_pkts[i]); 1321 1322 /* If tx burst fails drop slow packets */ 1323 for ( ; num_tx_slave < slave_slow_nb_pkts[i]; num_tx_slave++) 1324 rte_pktmbuf_free(slave_bufs[i][num_tx_slave]); 1325 1326 num_tx_total += num_tx_slave - slave_slow_nb_pkts[i]; 1327 num_tx_fail_total += slave_nb_pkts[i] - num_tx_slave; 1328 1329 /* If tx burst fails move packets to end of bufs */ 1330 if (unlikely(num_tx_slave < slave_nb_pkts[i])) { 1331 uint16_t j = nb_pkts - num_tx_fail_total; 1332 for ( ; num_tx_slave < slave_nb_pkts[i]; j++, num_tx_slave++) 1333 bufs[j] = slave_bufs[i][num_tx_slave]; 1334 } 1335 } 1336 1337 return num_tx_total; 1338 } 1339 1340 static uint16_t 1341 bond_ethdev_tx_burst_broadcast(void *queue, struct rte_mbuf **bufs, 1342 uint16_t nb_pkts) 1343 { 1344 struct bond_dev_private *internals; 1345 struct bond_tx_queue *bd_tx_q; 1346 1347 uint8_t tx_failed_flag = 0, num_of_slaves; 1348 uint16_t slaves[RTE_MAX_ETHPORTS]; 1349 1350 uint16_t max_nb_of_tx_pkts = 0; 1351 1352 int slave_tx_total[RTE_MAX_ETHPORTS]; 1353 int i, most_successful_tx_slave = -1; 1354 1355 bd_tx_q = (struct bond_tx_queue *)queue; 1356 internals = bd_tx_q->dev_private; 1357 1358 /* Copy slave list to protect against slave up/down changes during tx 1359 * bursting */ 1360 num_of_slaves = internals->active_slave_count; 1361 memcpy(slaves, internals->active_slaves, 1362 sizeof(internals->active_slaves[0]) * num_of_slaves); 1363 1364 if (num_of_slaves < 1) 1365 return 0; 1366 1367 /* Increment reference count on mbufs */ 1368 for (i = 0; i < nb_pkts; i++) 1369 rte_mbuf_refcnt_update(bufs[i], num_of_slaves - 1); 1370 1371 /* Transmit burst on each active slave */ 1372 for (i = 0; i < num_of_slaves; i++) { 1373 slave_tx_total[i] = rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id, 1374 bufs, nb_pkts); 1375 1376 if (unlikely(slave_tx_total[i] < nb_pkts)) 1377 tx_failed_flag = 1; 1378 1379 /* record the value and slave index for the slave which transmits the 1380 * maximum number of packets */ 1381 if (slave_tx_total[i] > max_nb_of_tx_pkts) { 1382 max_nb_of_tx_pkts = slave_tx_total[i]; 1383 most_successful_tx_slave = i; 1384 } 1385 } 1386 1387 /* if slaves fail to transmit packets from burst, the calling application 1388 * is not expected to know about multiple references to packets so we must 1389 * handle failures of all packets except those of the most successful slave 1390 */ 1391 if (unlikely(tx_failed_flag)) 1392 for (i = 0; i < num_of_slaves; i++) 1393 if (i != most_successful_tx_slave) 1394 while (slave_tx_total[i] < nb_pkts) 1395 rte_pktmbuf_free(bufs[slave_tx_total[i]++]); 1396 1397 return max_nb_of_tx_pkts; 1398 } 1399 1400 void 1401 link_properties_set(struct rte_eth_dev *ethdev, struct rte_eth_link *slave_link) 1402 { 1403 struct bond_dev_private *bond_ctx = ethdev->data->dev_private; 1404 1405 if (bond_ctx->mode == BONDING_MODE_8023AD) { 1406 /** 1407 * If in mode 4 then save the link properties of the first 1408 * slave, all subsequent slaves must match these properties 1409 */ 1410 struct rte_eth_link *bond_link = &bond_ctx->mode4.slave_link; 1411 1412 bond_link->link_autoneg = slave_link->link_autoneg; 1413 bond_link->link_duplex = slave_link->link_duplex; 1414 bond_link->link_speed = slave_link->link_speed; 1415 } else { 1416 /** 1417 * In any other mode the link properties are set to default 1418 * values of AUTONEG/DUPLEX 1419 */ 1420 ethdev->data->dev_link.link_autoneg = ETH_LINK_AUTONEG; 1421 ethdev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX; 1422 } 1423 } 1424 1425 int 1426 link_properties_valid(struct rte_eth_dev *ethdev, 1427 struct rte_eth_link *slave_link) 1428 { 1429 struct bond_dev_private *bond_ctx = ethdev->data->dev_private; 1430 1431 if (bond_ctx->mode == BONDING_MODE_8023AD) { 1432 struct rte_eth_link *bond_link = &bond_ctx->mode4.slave_link; 1433 1434 if (bond_link->link_duplex != slave_link->link_duplex || 1435 bond_link->link_autoneg != slave_link->link_autoneg || 1436 bond_link->link_speed != slave_link->link_speed) 1437 return -1; 1438 } 1439 1440 return 0; 1441 } 1442 1443 int 1444 mac_address_get(struct rte_eth_dev *eth_dev, struct ether_addr *dst_mac_addr) 1445 { 1446 struct ether_addr *mac_addr; 1447 1448 if (eth_dev == NULL) { 1449 RTE_LOG(ERR, PMD, "%s: NULL pointer eth_dev specified\n", __func__); 1450 return -1; 1451 } 1452 1453 if (dst_mac_addr == NULL) { 1454 RTE_LOG(ERR, PMD, "%s: NULL pointer MAC specified\n", __func__); 1455 return -1; 1456 } 1457 1458 mac_addr = eth_dev->data->mac_addrs; 1459 1460 ether_addr_copy(mac_addr, dst_mac_addr); 1461 return 0; 1462 } 1463 1464 int 1465 mac_address_set(struct rte_eth_dev *eth_dev, struct ether_addr *new_mac_addr) 1466 { 1467 struct ether_addr *mac_addr; 1468 1469 if (eth_dev == NULL) { 1470 RTE_BOND_LOG(ERR, "NULL pointer eth_dev specified"); 1471 return -1; 1472 } 1473 1474 if (new_mac_addr == NULL) { 1475 RTE_BOND_LOG(ERR, "NULL pointer MAC specified"); 1476 return -1; 1477 } 1478 1479 mac_addr = eth_dev->data->mac_addrs; 1480 1481 /* If new MAC is different to current MAC then update */ 1482 if (memcmp(mac_addr, new_mac_addr, sizeof(*mac_addr)) != 0) 1483 memcpy(mac_addr, new_mac_addr, sizeof(*mac_addr)); 1484 1485 return 0; 1486 } 1487 1488 int 1489 mac_address_slaves_update(struct rte_eth_dev *bonded_eth_dev) 1490 { 1491 struct bond_dev_private *internals = bonded_eth_dev->data->dev_private; 1492 int i; 1493 1494 /* Update slave devices MAC addresses */ 1495 if (internals->slave_count < 1) 1496 return -1; 1497 1498 switch (internals->mode) { 1499 case BONDING_MODE_ROUND_ROBIN: 1500 case BONDING_MODE_BALANCE: 1501 case BONDING_MODE_BROADCAST: 1502 for (i = 0; i < internals->slave_count; i++) { 1503 if (rte_eth_dev_default_mac_addr_set( 1504 internals->slaves[i].port_id, 1505 bonded_eth_dev->data->mac_addrs)) { 1506 RTE_BOND_LOG(ERR, "Failed to update port Id %d MAC address", 1507 internals->slaves[i].port_id); 1508 return -1; 1509 } 1510 } 1511 break; 1512 case BONDING_MODE_8023AD: 1513 bond_mode_8023ad_mac_address_update(bonded_eth_dev); 1514 break; 1515 case BONDING_MODE_ACTIVE_BACKUP: 1516 case BONDING_MODE_TLB: 1517 case BONDING_MODE_ALB: 1518 default: 1519 for (i = 0; i < internals->slave_count; i++) { 1520 if (internals->slaves[i].port_id == 1521 internals->current_primary_port) { 1522 if (rte_eth_dev_default_mac_addr_set( 1523 internals->primary_port, 1524 bonded_eth_dev->data->mac_addrs)) { 1525 RTE_BOND_LOG(ERR, "Failed to update port Id %d MAC address", 1526 internals->current_primary_port); 1527 return -1; 1528 } 1529 } else { 1530 if (rte_eth_dev_default_mac_addr_set( 1531 internals->slaves[i].port_id, 1532 &internals->slaves[i].persisted_mac_addr)) { 1533 RTE_BOND_LOG(ERR, "Failed to update port Id %d MAC address", 1534 internals->slaves[i].port_id); 1535 return -1; 1536 } 1537 } 1538 } 1539 } 1540 1541 return 0; 1542 } 1543 1544 int 1545 bond_ethdev_mode_set(struct rte_eth_dev *eth_dev, int mode) 1546 { 1547 struct bond_dev_private *internals; 1548 1549 internals = eth_dev->data->dev_private; 1550 1551 switch (mode) { 1552 case BONDING_MODE_ROUND_ROBIN: 1553 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_round_robin; 1554 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst; 1555 break; 1556 case BONDING_MODE_ACTIVE_BACKUP: 1557 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_active_backup; 1558 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst_active_backup; 1559 break; 1560 case BONDING_MODE_BALANCE: 1561 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_balance; 1562 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst; 1563 break; 1564 case BONDING_MODE_BROADCAST: 1565 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_broadcast; 1566 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst; 1567 break; 1568 case BONDING_MODE_8023AD: 1569 if (bond_mode_8023ad_enable(eth_dev) != 0) 1570 return -1; 1571 1572 if (internals->mode4.dedicated_queues.enabled == 0) { 1573 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst_8023ad; 1574 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_8023ad; 1575 RTE_LOG(WARNING, PMD, 1576 "Using mode 4, it is necessary to do TX burst " 1577 "and RX burst at least every 100ms.\n"); 1578 } else { 1579 /* Use flow director's optimization */ 1580 eth_dev->rx_pkt_burst = 1581 bond_ethdev_rx_burst_8023ad_fast_queue; 1582 eth_dev->tx_pkt_burst = 1583 bond_ethdev_tx_burst_8023ad_fast_queue; 1584 } 1585 break; 1586 case BONDING_MODE_TLB: 1587 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_tlb; 1588 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst_active_backup; 1589 break; 1590 case BONDING_MODE_ALB: 1591 if (bond_mode_alb_enable(eth_dev) != 0) 1592 return -1; 1593 1594 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_alb; 1595 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst_alb; 1596 break; 1597 default: 1598 return -1; 1599 } 1600 1601 internals->mode = mode; 1602 1603 return 0; 1604 } 1605 1606 1607 static int 1608 slave_configure_slow_queue(struct rte_eth_dev *bonded_eth_dev, 1609 struct rte_eth_dev *slave_eth_dev) 1610 { 1611 int errval = 0; 1612 struct bond_dev_private *internals = (struct bond_dev_private *) 1613 bonded_eth_dev->data->dev_private; 1614 struct port *port = &mode_8023ad_ports[slave_eth_dev->data->port_id]; 1615 1616 if (port->slow_pool == NULL) { 1617 char mem_name[256]; 1618 int slave_id = slave_eth_dev->data->port_id; 1619 1620 snprintf(mem_name, RTE_DIM(mem_name), "slave_port%u_slow_pool", 1621 slave_id); 1622 port->slow_pool = rte_pktmbuf_pool_create(mem_name, 8191, 1623 250, 0, RTE_MBUF_DEFAULT_BUF_SIZE, 1624 slave_eth_dev->data->numa_node); 1625 1626 /* Any memory allocation failure in initialization is critical because 1627 * resources can't be free, so reinitialization is impossible. */ 1628 if (port->slow_pool == NULL) { 1629 rte_panic("Slave %u: Failed to create memory pool '%s': %s\n", 1630 slave_id, mem_name, rte_strerror(rte_errno)); 1631 } 1632 } 1633 1634 if (internals->mode4.dedicated_queues.enabled == 1) { 1635 /* Configure slow Rx queue */ 1636 1637 errval = rte_eth_rx_queue_setup(slave_eth_dev->data->port_id, 1638 internals->mode4.dedicated_queues.rx_qid, 128, 1639 rte_eth_dev_socket_id(slave_eth_dev->data->port_id), 1640 NULL, port->slow_pool); 1641 if (errval != 0) { 1642 RTE_BOND_LOG(ERR, 1643 "rte_eth_rx_queue_setup: port=%d queue_id %d, err (%d)", 1644 slave_eth_dev->data->port_id, 1645 internals->mode4.dedicated_queues.rx_qid, 1646 errval); 1647 return errval; 1648 } 1649 1650 errval = rte_eth_tx_queue_setup(slave_eth_dev->data->port_id, 1651 internals->mode4.dedicated_queues.tx_qid, 512, 1652 rte_eth_dev_socket_id(slave_eth_dev->data->port_id), 1653 NULL); 1654 if (errval != 0) { 1655 RTE_BOND_LOG(ERR, 1656 "rte_eth_tx_queue_setup: port=%d queue_id %d, err (%d)", 1657 slave_eth_dev->data->port_id, 1658 internals->mode4.dedicated_queues.tx_qid, 1659 errval); 1660 return errval; 1661 } 1662 } 1663 return 0; 1664 } 1665 1666 int 1667 slave_configure(struct rte_eth_dev *bonded_eth_dev, 1668 struct rte_eth_dev *slave_eth_dev) 1669 { 1670 struct bond_rx_queue *bd_rx_q; 1671 struct bond_tx_queue *bd_tx_q; 1672 uint16_t nb_rx_queues; 1673 uint16_t nb_tx_queues; 1674 1675 int errval; 1676 uint16_t q_id; 1677 struct rte_flow_error flow_error; 1678 1679 struct bond_dev_private *internals = (struct bond_dev_private *) 1680 bonded_eth_dev->data->dev_private; 1681 1682 /* Stop slave */ 1683 rte_eth_dev_stop(slave_eth_dev->data->port_id); 1684 1685 /* Enable interrupts on slave device if supported */ 1686 if (slave_eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) 1687 slave_eth_dev->data->dev_conf.intr_conf.lsc = 1; 1688 1689 /* If RSS is enabled for bonding, try to enable it for slaves */ 1690 if (bonded_eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) { 1691 if (bonded_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len 1692 != 0) { 1693 slave_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len = 1694 bonded_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len; 1695 slave_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key = 1696 bonded_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key; 1697 } else { 1698 slave_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 1699 } 1700 1701 slave_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf = 1702 bonded_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf; 1703 slave_eth_dev->data->dev_conf.rxmode.mq_mode = 1704 bonded_eth_dev->data->dev_conf.rxmode.mq_mode; 1705 } 1706 1707 slave_eth_dev->data->dev_conf.rxmode.hw_vlan_filter = 1708 bonded_eth_dev->data->dev_conf.rxmode.hw_vlan_filter; 1709 1710 nb_rx_queues = bonded_eth_dev->data->nb_rx_queues; 1711 nb_tx_queues = bonded_eth_dev->data->nb_tx_queues; 1712 1713 if (internals->mode == BONDING_MODE_8023AD) { 1714 if (internals->mode4.dedicated_queues.enabled == 1) { 1715 nb_rx_queues++; 1716 nb_tx_queues++; 1717 } 1718 } 1719 1720 /* Configure device */ 1721 errval = rte_eth_dev_configure(slave_eth_dev->data->port_id, 1722 nb_rx_queues, nb_tx_queues, 1723 &(slave_eth_dev->data->dev_conf)); 1724 if (errval != 0) { 1725 RTE_BOND_LOG(ERR, "Cannot configure slave device: port %u , err (%d)", 1726 slave_eth_dev->data->port_id, errval); 1727 return errval; 1728 } 1729 1730 /* Setup Rx Queues */ 1731 for (q_id = 0; q_id < bonded_eth_dev->data->nb_rx_queues; q_id++) { 1732 bd_rx_q = (struct bond_rx_queue *)bonded_eth_dev->data->rx_queues[q_id]; 1733 1734 errval = rte_eth_rx_queue_setup(slave_eth_dev->data->port_id, q_id, 1735 bd_rx_q->nb_rx_desc, 1736 rte_eth_dev_socket_id(slave_eth_dev->data->port_id), 1737 &(bd_rx_q->rx_conf), bd_rx_q->mb_pool); 1738 if (errval != 0) { 1739 RTE_BOND_LOG(ERR, 1740 "rte_eth_rx_queue_setup: port=%d queue_id %d, err (%d)", 1741 slave_eth_dev->data->port_id, q_id, errval); 1742 return errval; 1743 } 1744 } 1745 1746 /* Setup Tx Queues */ 1747 for (q_id = 0; q_id < bonded_eth_dev->data->nb_tx_queues; q_id++) { 1748 bd_tx_q = (struct bond_tx_queue *)bonded_eth_dev->data->tx_queues[q_id]; 1749 1750 errval = rte_eth_tx_queue_setup(slave_eth_dev->data->port_id, q_id, 1751 bd_tx_q->nb_tx_desc, 1752 rte_eth_dev_socket_id(slave_eth_dev->data->port_id), 1753 &bd_tx_q->tx_conf); 1754 if (errval != 0) { 1755 RTE_BOND_LOG(ERR, 1756 "rte_eth_tx_queue_setup: port=%d queue_id %d, err (%d)", 1757 slave_eth_dev->data->port_id, q_id, errval); 1758 return errval; 1759 } 1760 } 1761 1762 if (internals->mode == BONDING_MODE_8023AD && 1763 internals->mode4.dedicated_queues.enabled == 1) { 1764 if (slave_configure_slow_queue(bonded_eth_dev, slave_eth_dev) 1765 != 0) 1766 return errval; 1767 1768 if (bond_ethdev_8023ad_flow_verify(bonded_eth_dev, 1769 slave_eth_dev->data->port_id) != 0) { 1770 RTE_BOND_LOG(ERR, 1771 "rte_eth_tx_queue_setup: port=%d queue_id %d, err (%d)", 1772 slave_eth_dev->data->port_id, q_id, errval); 1773 return -1; 1774 } 1775 1776 if (internals->mode4.dedicated_queues.flow[slave_eth_dev->data->port_id] != NULL) 1777 rte_flow_destroy(slave_eth_dev->data->port_id, 1778 internals->mode4.dedicated_queues.flow[slave_eth_dev->data->port_id], 1779 &flow_error); 1780 1781 bond_ethdev_8023ad_flow_set(bonded_eth_dev, 1782 slave_eth_dev->data->port_id); 1783 } 1784 1785 /* Start device */ 1786 errval = rte_eth_dev_start(slave_eth_dev->data->port_id); 1787 if (errval != 0) { 1788 RTE_BOND_LOG(ERR, "rte_eth_dev_start: port=%u, err (%d)", 1789 slave_eth_dev->data->port_id, errval); 1790 return -1; 1791 } 1792 1793 /* If RSS is enabled for bonding, synchronize RETA */ 1794 if (bonded_eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS) { 1795 int i; 1796 struct bond_dev_private *internals; 1797 1798 internals = bonded_eth_dev->data->dev_private; 1799 1800 for (i = 0; i < internals->slave_count; i++) { 1801 if (internals->slaves[i].port_id == slave_eth_dev->data->port_id) { 1802 errval = rte_eth_dev_rss_reta_update( 1803 slave_eth_dev->data->port_id, 1804 &internals->reta_conf[0], 1805 internals->slaves[i].reta_size); 1806 if (errval != 0) { 1807 RTE_LOG(WARNING, PMD, 1808 "rte_eth_dev_rss_reta_update on slave port %d fails (err %d)." 1809 " RSS Configuration for bonding may be inconsistent.\n", 1810 slave_eth_dev->data->port_id, errval); 1811 } 1812 break; 1813 } 1814 } 1815 } 1816 1817 /* If lsc interrupt is set, check initial slave's link status */ 1818 if (slave_eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) { 1819 slave_eth_dev->dev_ops->link_update(slave_eth_dev, 0); 1820 bond_ethdev_lsc_event_callback(slave_eth_dev->data->port_id, 1821 RTE_ETH_EVENT_INTR_LSC, &bonded_eth_dev->data->port_id, 1822 NULL); 1823 } 1824 1825 return 0; 1826 } 1827 1828 void 1829 slave_remove(struct bond_dev_private *internals, 1830 struct rte_eth_dev *slave_eth_dev) 1831 { 1832 uint8_t i; 1833 1834 for (i = 0; i < internals->slave_count; i++) 1835 if (internals->slaves[i].port_id == 1836 slave_eth_dev->data->port_id) 1837 break; 1838 1839 if (i < (internals->slave_count - 1)) 1840 memmove(&internals->slaves[i], &internals->slaves[i + 1], 1841 sizeof(internals->slaves[0]) * 1842 (internals->slave_count - i - 1)); 1843 1844 internals->slave_count--; 1845 1846 /* force reconfiguration of slave interfaces */ 1847 _rte_eth_dev_reset(slave_eth_dev); 1848 } 1849 1850 static void 1851 bond_ethdev_slave_link_status_change_monitor(void *cb_arg); 1852 1853 void 1854 slave_add(struct bond_dev_private *internals, 1855 struct rte_eth_dev *slave_eth_dev) 1856 { 1857 struct bond_slave_details *slave_details = 1858 &internals->slaves[internals->slave_count]; 1859 1860 slave_details->port_id = slave_eth_dev->data->port_id; 1861 slave_details->last_link_status = 0; 1862 1863 /* Mark slave devices that don't support interrupts so we can 1864 * compensate when we start the bond 1865 */ 1866 if (!(slave_eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)) { 1867 slave_details->link_status_poll_enabled = 1; 1868 } 1869 1870 slave_details->link_status_wait_to_complete = 0; 1871 /* clean tlb_last_obytes when adding port for bonding device */ 1872 memcpy(&(slave_details->persisted_mac_addr), slave_eth_dev->data->mac_addrs, 1873 sizeof(struct ether_addr)); 1874 } 1875 1876 void 1877 bond_ethdev_primary_set(struct bond_dev_private *internals, 1878 uint16_t slave_port_id) 1879 { 1880 int i; 1881 1882 if (internals->active_slave_count < 1) 1883 internals->current_primary_port = slave_port_id; 1884 else 1885 /* Search bonded device slave ports for new proposed primary port */ 1886 for (i = 0; i < internals->active_slave_count; i++) { 1887 if (internals->active_slaves[i] == slave_port_id) 1888 internals->current_primary_port = slave_port_id; 1889 } 1890 } 1891 1892 static void 1893 bond_ethdev_promiscuous_enable(struct rte_eth_dev *eth_dev); 1894 1895 static int 1896 bond_ethdev_start(struct rte_eth_dev *eth_dev) 1897 { 1898 struct bond_dev_private *internals; 1899 int i; 1900 1901 /* slave eth dev will be started by bonded device */ 1902 if (check_for_bonded_ethdev(eth_dev)) { 1903 RTE_BOND_LOG(ERR, "User tried to explicitly start a slave eth_dev (%d)", 1904 eth_dev->data->port_id); 1905 return -1; 1906 } 1907 1908 eth_dev->data->dev_link.link_status = ETH_LINK_DOWN; 1909 eth_dev->data->dev_started = 1; 1910 1911 internals = eth_dev->data->dev_private; 1912 1913 if (internals->slave_count == 0) { 1914 RTE_BOND_LOG(ERR, "Cannot start port since there are no slave devices"); 1915 goto out_err; 1916 } 1917 1918 if (internals->user_defined_mac == 0) { 1919 struct ether_addr *new_mac_addr = NULL; 1920 1921 for (i = 0; i < internals->slave_count; i++) 1922 if (internals->slaves[i].port_id == internals->primary_port) 1923 new_mac_addr = &internals->slaves[i].persisted_mac_addr; 1924 1925 if (new_mac_addr == NULL) 1926 goto out_err; 1927 1928 if (mac_address_set(eth_dev, new_mac_addr) != 0) { 1929 RTE_BOND_LOG(ERR, "bonded port (%d) failed to update MAC address", 1930 eth_dev->data->port_id); 1931 goto out_err; 1932 } 1933 } 1934 1935 /* If bonded device is configure in promiscuous mode then re-apply config */ 1936 if (internals->promiscuous_en) 1937 bond_ethdev_promiscuous_enable(eth_dev); 1938 1939 if (internals->mode == BONDING_MODE_8023AD) { 1940 if (internals->mode4.dedicated_queues.enabled == 1) { 1941 internals->mode4.dedicated_queues.rx_qid = 1942 eth_dev->data->nb_rx_queues; 1943 internals->mode4.dedicated_queues.tx_qid = 1944 eth_dev->data->nb_tx_queues; 1945 } 1946 } 1947 1948 1949 /* Reconfigure each slave device if starting bonded device */ 1950 for (i = 0; i < internals->slave_count; i++) { 1951 struct rte_eth_dev *slave_ethdev = 1952 &(rte_eth_devices[internals->slaves[i].port_id]); 1953 if (slave_configure(eth_dev, slave_ethdev) != 0) { 1954 RTE_BOND_LOG(ERR, 1955 "bonded port (%d) failed to reconfigure slave device (%d)", 1956 eth_dev->data->port_id, 1957 internals->slaves[i].port_id); 1958 goto out_err; 1959 } 1960 /* We will need to poll for link status if any slave doesn't 1961 * support interrupts 1962 */ 1963 if (internals->slaves[i].link_status_poll_enabled) 1964 internals->link_status_polling_enabled = 1; 1965 } 1966 1967 /* start polling if needed */ 1968 if (internals->link_status_polling_enabled) { 1969 rte_eal_alarm_set( 1970 internals->link_status_polling_interval_ms * 1000, 1971 bond_ethdev_slave_link_status_change_monitor, 1972 (void *)&rte_eth_devices[internals->port_id]); 1973 } 1974 1975 /* Update all slave devices MACs*/ 1976 if (mac_address_slaves_update(eth_dev) != 0) 1977 goto out_err; 1978 1979 if (internals->user_defined_primary_port) 1980 bond_ethdev_primary_set(internals, internals->primary_port); 1981 1982 if (internals->mode == BONDING_MODE_8023AD) 1983 bond_mode_8023ad_start(eth_dev); 1984 1985 if (internals->mode == BONDING_MODE_TLB || 1986 internals->mode == BONDING_MODE_ALB) 1987 bond_tlb_enable(internals); 1988 1989 return 0; 1990 1991 out_err: 1992 eth_dev->data->dev_started = 0; 1993 return -1; 1994 } 1995 1996 static void 1997 bond_ethdev_free_queues(struct rte_eth_dev *dev) 1998 { 1999 uint8_t i; 2000 2001 if (dev->data->rx_queues != NULL) { 2002 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2003 rte_free(dev->data->rx_queues[i]); 2004 dev->data->rx_queues[i] = NULL; 2005 } 2006 dev->data->nb_rx_queues = 0; 2007 } 2008 2009 if (dev->data->tx_queues != NULL) { 2010 for (i = 0; i < dev->data->nb_tx_queues; i++) { 2011 rte_free(dev->data->tx_queues[i]); 2012 dev->data->tx_queues[i] = NULL; 2013 } 2014 dev->data->nb_tx_queues = 0; 2015 } 2016 } 2017 2018 void 2019 bond_ethdev_stop(struct rte_eth_dev *eth_dev) 2020 { 2021 struct bond_dev_private *internals = eth_dev->data->dev_private; 2022 uint8_t i; 2023 2024 if (internals->mode == BONDING_MODE_8023AD) { 2025 struct port *port; 2026 void *pkt = NULL; 2027 2028 bond_mode_8023ad_stop(eth_dev); 2029 2030 /* Discard all messages to/from mode 4 state machines */ 2031 for (i = 0; i < internals->active_slave_count; i++) { 2032 port = &mode_8023ad_ports[internals->active_slaves[i]]; 2033 2034 RTE_ASSERT(port->rx_ring != NULL); 2035 while (rte_ring_dequeue(port->rx_ring, &pkt) != -ENOENT) 2036 rte_pktmbuf_free(pkt); 2037 2038 RTE_ASSERT(port->tx_ring != NULL); 2039 while (rte_ring_dequeue(port->tx_ring, &pkt) != -ENOENT) 2040 rte_pktmbuf_free(pkt); 2041 } 2042 } 2043 2044 if (internals->mode == BONDING_MODE_TLB || 2045 internals->mode == BONDING_MODE_ALB) { 2046 bond_tlb_disable(internals); 2047 for (i = 0; i < internals->active_slave_count; i++) 2048 tlb_last_obytets[internals->active_slaves[i]] = 0; 2049 } 2050 2051 internals->link_status_polling_enabled = 0; 2052 for (i = 0; i < internals->slave_count; i++) 2053 internals->slaves[i].last_link_status = 0; 2054 2055 eth_dev->data->dev_link.link_status = ETH_LINK_DOWN; 2056 eth_dev->data->dev_started = 0; 2057 } 2058 2059 void 2060 bond_ethdev_close(struct rte_eth_dev *dev) 2061 { 2062 struct bond_dev_private *internals = dev->data->dev_private; 2063 uint8_t bond_port_id = internals->port_id; 2064 int skipped = 0; 2065 2066 RTE_LOG(INFO, EAL, "Closing bonded device %s\n", dev->device->name); 2067 while (internals->slave_count != skipped) { 2068 uint16_t port_id = internals->slaves[skipped].port_id; 2069 2070 rte_eth_dev_stop(port_id); 2071 2072 if (rte_eth_bond_slave_remove(bond_port_id, port_id) != 0) { 2073 RTE_LOG(ERR, EAL, 2074 "Failed to remove port %d from bonded device " 2075 "%s\n", port_id, dev->device->name); 2076 skipped++; 2077 } 2078 } 2079 bond_ethdev_free_queues(dev); 2080 rte_bitmap_reset(internals->vlan_filter_bmp); 2081 } 2082 2083 /* forward declaration */ 2084 static int bond_ethdev_configure(struct rte_eth_dev *dev); 2085 2086 static void 2087 bond_ethdev_info(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 2088 { 2089 struct bond_dev_private *internals = dev->data->dev_private; 2090 2091 uint16_t max_nb_rx_queues = UINT16_MAX; 2092 uint16_t max_nb_tx_queues = UINT16_MAX; 2093 2094 dev_info->max_mac_addrs = 1; 2095 2096 dev_info->max_rx_pktlen = internals->candidate_max_rx_pktlen ? 2097 internals->candidate_max_rx_pktlen : 2098 ETHER_MAX_JUMBO_FRAME_LEN; 2099 2100 /* Max number of tx/rx queues that the bonded device can support is the 2101 * minimum values of the bonded slaves, as all slaves must be capable 2102 * of supporting the same number of tx/rx queues. 2103 */ 2104 if (internals->slave_count > 0) { 2105 struct rte_eth_dev_info slave_info; 2106 uint8_t idx; 2107 2108 for (idx = 0; idx < internals->slave_count; idx++) { 2109 rte_eth_dev_info_get(internals->slaves[idx].port_id, 2110 &slave_info); 2111 2112 if (slave_info.max_rx_queues < max_nb_rx_queues) 2113 max_nb_rx_queues = slave_info.max_rx_queues; 2114 2115 if (slave_info.max_tx_queues < max_nb_tx_queues) 2116 max_nb_tx_queues = slave_info.max_tx_queues; 2117 } 2118 } 2119 2120 dev_info->max_rx_queues = max_nb_rx_queues; 2121 dev_info->max_tx_queues = max_nb_tx_queues; 2122 2123 /** 2124 * If dedicated hw queues enabled for link bonding device in LACP mode 2125 * then we need to reduce the maximum number of data path queues by 1. 2126 */ 2127 if (internals->mode == BONDING_MODE_8023AD && 2128 internals->mode4.dedicated_queues.enabled == 1) { 2129 dev_info->max_rx_queues--; 2130 dev_info->max_tx_queues--; 2131 } 2132 2133 dev_info->min_rx_bufsize = 0; 2134 2135 dev_info->rx_offload_capa = internals->rx_offload_capa; 2136 dev_info->tx_offload_capa = internals->tx_offload_capa; 2137 dev_info->flow_type_rss_offloads = internals->flow_type_rss_offloads; 2138 2139 dev_info->reta_size = internals->reta_size; 2140 } 2141 2142 static int 2143 bond_ethdev_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 2144 { 2145 int res; 2146 uint16_t i; 2147 struct bond_dev_private *internals = dev->data->dev_private; 2148 2149 /* don't do this while a slave is being added */ 2150 rte_spinlock_lock(&internals->lock); 2151 2152 if (on) 2153 rte_bitmap_set(internals->vlan_filter_bmp, vlan_id); 2154 else 2155 rte_bitmap_clear(internals->vlan_filter_bmp, vlan_id); 2156 2157 for (i = 0; i < internals->slave_count; i++) { 2158 uint16_t port_id = internals->slaves[i].port_id; 2159 2160 res = rte_eth_dev_vlan_filter(port_id, vlan_id, on); 2161 if (res == ENOTSUP) 2162 RTE_LOG(WARNING, PMD, 2163 "Setting VLAN filter on slave port %u not supported.\n", 2164 port_id); 2165 } 2166 2167 rte_spinlock_unlock(&internals->lock); 2168 return 0; 2169 } 2170 2171 static int 2172 bond_ethdev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id, 2173 uint16_t nb_rx_desc, unsigned int socket_id __rte_unused, 2174 const struct rte_eth_rxconf *rx_conf, struct rte_mempool *mb_pool) 2175 { 2176 struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *) 2177 rte_zmalloc_socket(NULL, sizeof(struct bond_rx_queue), 2178 0, dev->data->numa_node); 2179 if (bd_rx_q == NULL) 2180 return -1; 2181 2182 bd_rx_q->queue_id = rx_queue_id; 2183 bd_rx_q->dev_private = dev->data->dev_private; 2184 2185 bd_rx_q->nb_rx_desc = nb_rx_desc; 2186 2187 memcpy(&(bd_rx_q->rx_conf), rx_conf, sizeof(struct rte_eth_rxconf)); 2188 bd_rx_q->mb_pool = mb_pool; 2189 2190 dev->data->rx_queues[rx_queue_id] = bd_rx_q; 2191 2192 return 0; 2193 } 2194 2195 static int 2196 bond_ethdev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id, 2197 uint16_t nb_tx_desc, unsigned int socket_id __rte_unused, 2198 const struct rte_eth_txconf *tx_conf) 2199 { 2200 struct bond_tx_queue *bd_tx_q = (struct bond_tx_queue *) 2201 rte_zmalloc_socket(NULL, sizeof(struct bond_tx_queue), 2202 0, dev->data->numa_node); 2203 2204 if (bd_tx_q == NULL) 2205 return -1; 2206 2207 bd_tx_q->queue_id = tx_queue_id; 2208 bd_tx_q->dev_private = dev->data->dev_private; 2209 2210 bd_tx_q->nb_tx_desc = nb_tx_desc; 2211 memcpy(&(bd_tx_q->tx_conf), tx_conf, sizeof(bd_tx_q->tx_conf)); 2212 2213 dev->data->tx_queues[tx_queue_id] = bd_tx_q; 2214 2215 return 0; 2216 } 2217 2218 static void 2219 bond_ethdev_rx_queue_release(void *queue) 2220 { 2221 if (queue == NULL) 2222 return; 2223 2224 rte_free(queue); 2225 } 2226 2227 static void 2228 bond_ethdev_tx_queue_release(void *queue) 2229 { 2230 if (queue == NULL) 2231 return; 2232 2233 rte_free(queue); 2234 } 2235 2236 static void 2237 bond_ethdev_slave_link_status_change_monitor(void *cb_arg) 2238 { 2239 struct rte_eth_dev *bonded_ethdev, *slave_ethdev; 2240 struct bond_dev_private *internals; 2241 2242 /* Default value for polling slave found is true as we don't want to 2243 * disable the polling thread if we cannot get the lock */ 2244 int i, polling_slave_found = 1; 2245 2246 if (cb_arg == NULL) 2247 return; 2248 2249 bonded_ethdev = (struct rte_eth_dev *)cb_arg; 2250 internals = (struct bond_dev_private *)bonded_ethdev->data->dev_private; 2251 2252 if (!bonded_ethdev->data->dev_started || 2253 !internals->link_status_polling_enabled) 2254 return; 2255 2256 /* If device is currently being configured then don't check slaves link 2257 * status, wait until next period */ 2258 if (rte_spinlock_trylock(&internals->lock)) { 2259 if (internals->slave_count > 0) 2260 polling_slave_found = 0; 2261 2262 for (i = 0; i < internals->slave_count; i++) { 2263 if (!internals->slaves[i].link_status_poll_enabled) 2264 continue; 2265 2266 slave_ethdev = &rte_eth_devices[internals->slaves[i].port_id]; 2267 polling_slave_found = 1; 2268 2269 /* Update slave link status */ 2270 (*slave_ethdev->dev_ops->link_update)(slave_ethdev, 2271 internals->slaves[i].link_status_wait_to_complete); 2272 2273 /* if link status has changed since last checked then call lsc 2274 * event callback */ 2275 if (slave_ethdev->data->dev_link.link_status != 2276 internals->slaves[i].last_link_status) { 2277 internals->slaves[i].last_link_status = 2278 slave_ethdev->data->dev_link.link_status; 2279 2280 bond_ethdev_lsc_event_callback(internals->slaves[i].port_id, 2281 RTE_ETH_EVENT_INTR_LSC, 2282 &bonded_ethdev->data->port_id, 2283 NULL); 2284 } 2285 } 2286 rte_spinlock_unlock(&internals->lock); 2287 } 2288 2289 if (polling_slave_found) 2290 /* Set alarm to continue monitoring link status of slave ethdev's */ 2291 rte_eal_alarm_set(internals->link_status_polling_interval_ms * 1000, 2292 bond_ethdev_slave_link_status_change_monitor, cb_arg); 2293 } 2294 2295 static int 2296 bond_ethdev_link_update(struct rte_eth_dev *ethdev, int wait_to_complete) 2297 { 2298 void (*link_update)(uint16_t port_id, struct rte_eth_link *eth_link); 2299 2300 struct bond_dev_private *bond_ctx; 2301 struct rte_eth_link slave_link; 2302 2303 uint32_t idx; 2304 2305 bond_ctx = ethdev->data->dev_private; 2306 2307 ethdev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE; 2308 2309 if (ethdev->data->dev_started == 0 || 2310 bond_ctx->active_slave_count == 0) { 2311 ethdev->data->dev_link.link_status = ETH_LINK_DOWN; 2312 return 0; 2313 } 2314 2315 ethdev->data->dev_link.link_status = ETH_LINK_UP; 2316 2317 if (wait_to_complete) 2318 link_update = rte_eth_link_get; 2319 else 2320 link_update = rte_eth_link_get_nowait; 2321 2322 switch (bond_ctx->mode) { 2323 case BONDING_MODE_BROADCAST: 2324 /** 2325 * Setting link speed to UINT32_MAX to ensure we pick up the 2326 * value of the first active slave 2327 */ 2328 ethdev->data->dev_link.link_speed = UINT32_MAX; 2329 2330 /** 2331 * link speed is minimum value of all the slaves link speed as 2332 * packet loss will occur on this slave if transmission at rates 2333 * greater than this are attempted 2334 */ 2335 for (idx = 1; idx < bond_ctx->active_slave_count; idx++) { 2336 link_update(bond_ctx->active_slaves[0], &slave_link); 2337 2338 if (slave_link.link_speed < 2339 ethdev->data->dev_link.link_speed) 2340 ethdev->data->dev_link.link_speed = 2341 slave_link.link_speed; 2342 } 2343 break; 2344 case BONDING_MODE_ACTIVE_BACKUP: 2345 /* Current primary slave */ 2346 link_update(bond_ctx->current_primary_port, &slave_link); 2347 2348 ethdev->data->dev_link.link_speed = slave_link.link_speed; 2349 break; 2350 case BONDING_MODE_8023AD: 2351 ethdev->data->dev_link.link_autoneg = 2352 bond_ctx->mode4.slave_link.link_autoneg; 2353 ethdev->data->dev_link.link_duplex = 2354 bond_ctx->mode4.slave_link.link_duplex; 2355 /* fall through to update link speed */ 2356 case BONDING_MODE_ROUND_ROBIN: 2357 case BONDING_MODE_BALANCE: 2358 case BONDING_MODE_TLB: 2359 case BONDING_MODE_ALB: 2360 default: 2361 /** 2362 * In theses mode the maximum theoretical link speed is the sum 2363 * of all the slaves 2364 */ 2365 ethdev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE; 2366 2367 for (idx = 0; idx < bond_ctx->active_slave_count; idx++) { 2368 link_update(bond_ctx->active_slaves[idx], &slave_link); 2369 2370 ethdev->data->dev_link.link_speed += 2371 slave_link.link_speed; 2372 } 2373 } 2374 2375 2376 return 0; 2377 } 2378 2379 2380 static int 2381 bond_ethdev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 2382 { 2383 struct bond_dev_private *internals = dev->data->dev_private; 2384 struct rte_eth_stats slave_stats; 2385 int i, j; 2386 2387 for (i = 0; i < internals->slave_count; i++) { 2388 rte_eth_stats_get(internals->slaves[i].port_id, &slave_stats); 2389 2390 stats->ipackets += slave_stats.ipackets; 2391 stats->opackets += slave_stats.opackets; 2392 stats->ibytes += slave_stats.ibytes; 2393 stats->obytes += slave_stats.obytes; 2394 stats->imissed += slave_stats.imissed; 2395 stats->ierrors += slave_stats.ierrors; 2396 stats->oerrors += slave_stats.oerrors; 2397 stats->rx_nombuf += slave_stats.rx_nombuf; 2398 2399 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 2400 stats->q_ipackets[j] += slave_stats.q_ipackets[j]; 2401 stats->q_opackets[j] += slave_stats.q_opackets[j]; 2402 stats->q_ibytes[j] += slave_stats.q_ibytes[j]; 2403 stats->q_obytes[j] += slave_stats.q_obytes[j]; 2404 stats->q_errors[j] += slave_stats.q_errors[j]; 2405 } 2406 2407 } 2408 2409 return 0; 2410 } 2411 2412 static void 2413 bond_ethdev_stats_reset(struct rte_eth_dev *dev) 2414 { 2415 struct bond_dev_private *internals = dev->data->dev_private; 2416 int i; 2417 2418 for (i = 0; i < internals->slave_count; i++) 2419 rte_eth_stats_reset(internals->slaves[i].port_id); 2420 } 2421 2422 static void 2423 bond_ethdev_promiscuous_enable(struct rte_eth_dev *eth_dev) 2424 { 2425 struct bond_dev_private *internals = eth_dev->data->dev_private; 2426 int i; 2427 2428 internals->promiscuous_en = 1; 2429 2430 switch (internals->mode) { 2431 /* Promiscuous mode is propagated to all slaves */ 2432 case BONDING_MODE_ROUND_ROBIN: 2433 case BONDING_MODE_BALANCE: 2434 case BONDING_MODE_BROADCAST: 2435 for (i = 0; i < internals->slave_count; i++) 2436 rte_eth_promiscuous_enable(internals->slaves[i].port_id); 2437 break; 2438 /* In mode4 promiscus mode is managed when slave is added/removed */ 2439 case BONDING_MODE_8023AD: 2440 break; 2441 /* Promiscuous mode is propagated only to primary slave */ 2442 case BONDING_MODE_ACTIVE_BACKUP: 2443 case BONDING_MODE_TLB: 2444 case BONDING_MODE_ALB: 2445 default: 2446 rte_eth_promiscuous_enable(internals->current_primary_port); 2447 } 2448 } 2449 2450 static void 2451 bond_ethdev_promiscuous_disable(struct rte_eth_dev *dev) 2452 { 2453 struct bond_dev_private *internals = dev->data->dev_private; 2454 int i; 2455 2456 internals->promiscuous_en = 0; 2457 2458 switch (internals->mode) { 2459 /* Promiscuous mode is propagated to all slaves */ 2460 case BONDING_MODE_ROUND_ROBIN: 2461 case BONDING_MODE_BALANCE: 2462 case BONDING_MODE_BROADCAST: 2463 for (i = 0; i < internals->slave_count; i++) 2464 rte_eth_promiscuous_disable(internals->slaves[i].port_id); 2465 break; 2466 /* In mode4 promiscus mode is set managed when slave is added/removed */ 2467 case BONDING_MODE_8023AD: 2468 break; 2469 /* Promiscuous mode is propagated only to primary slave */ 2470 case BONDING_MODE_ACTIVE_BACKUP: 2471 case BONDING_MODE_TLB: 2472 case BONDING_MODE_ALB: 2473 default: 2474 rte_eth_promiscuous_disable(internals->current_primary_port); 2475 } 2476 } 2477 2478 static void 2479 bond_ethdev_delayed_lsc_propagation(void *arg) 2480 { 2481 if (arg == NULL) 2482 return; 2483 2484 _rte_eth_dev_callback_process((struct rte_eth_dev *)arg, 2485 RTE_ETH_EVENT_INTR_LSC, NULL, NULL); 2486 } 2487 2488 int 2489 bond_ethdev_lsc_event_callback(uint16_t port_id, enum rte_eth_event_type type, 2490 void *param, void *ret_param __rte_unused) 2491 { 2492 struct rte_eth_dev *bonded_eth_dev; 2493 struct bond_dev_private *internals; 2494 struct rte_eth_link link; 2495 int rc = -1; 2496 2497 int i, valid_slave = 0; 2498 uint8_t active_pos; 2499 uint8_t lsc_flag = 0; 2500 2501 if (type != RTE_ETH_EVENT_INTR_LSC || param == NULL) 2502 return rc; 2503 2504 bonded_eth_dev = &rte_eth_devices[*(uint8_t *)param]; 2505 2506 if (check_for_bonded_ethdev(bonded_eth_dev)) 2507 return rc; 2508 2509 internals = bonded_eth_dev->data->dev_private; 2510 2511 /* If the device isn't started don't handle interrupts */ 2512 if (!bonded_eth_dev->data->dev_started) 2513 return rc; 2514 2515 /* verify that port_id is a valid slave of bonded port */ 2516 for (i = 0; i < internals->slave_count; i++) { 2517 if (internals->slaves[i].port_id == port_id) { 2518 valid_slave = 1; 2519 break; 2520 } 2521 } 2522 2523 if (!valid_slave) 2524 return rc; 2525 2526 /* Synchronize lsc callback parallel calls either by real link event 2527 * from the slaves PMDs or by the bonding PMD itself. 2528 */ 2529 rte_spinlock_lock(&internals->lsc_lock); 2530 2531 /* Search for port in active port list */ 2532 active_pos = find_slave_by_id(internals->active_slaves, 2533 internals->active_slave_count, port_id); 2534 2535 rte_eth_link_get_nowait(port_id, &link); 2536 if (link.link_status) { 2537 if (active_pos < internals->active_slave_count) 2538 goto link_update; 2539 2540 /* if no active slave ports then set this port to be primary port */ 2541 if (internals->active_slave_count < 1) { 2542 /* If first active slave, then change link status */ 2543 bonded_eth_dev->data->dev_link.link_status = ETH_LINK_UP; 2544 internals->current_primary_port = port_id; 2545 lsc_flag = 1; 2546 2547 mac_address_slaves_update(bonded_eth_dev); 2548 } 2549 2550 activate_slave(bonded_eth_dev, port_id); 2551 2552 /* If user has defined the primary port then default to using it */ 2553 if (internals->user_defined_primary_port && 2554 internals->primary_port == port_id) 2555 bond_ethdev_primary_set(internals, port_id); 2556 } else { 2557 if (active_pos == internals->active_slave_count) 2558 goto link_update; 2559 2560 /* Remove from active slave list */ 2561 deactivate_slave(bonded_eth_dev, port_id); 2562 2563 if (internals->active_slave_count < 1) 2564 lsc_flag = 1; 2565 2566 /* Update primary id, take first active slave from list or if none 2567 * available set to -1 */ 2568 if (port_id == internals->current_primary_port) { 2569 if (internals->active_slave_count > 0) 2570 bond_ethdev_primary_set(internals, 2571 internals->active_slaves[0]); 2572 else 2573 internals->current_primary_port = internals->primary_port; 2574 } 2575 } 2576 2577 link_update: 2578 /** 2579 * Update bonded device link properties after any change to active 2580 * slaves 2581 */ 2582 bond_ethdev_link_update(bonded_eth_dev, 0); 2583 2584 if (lsc_flag) { 2585 /* Cancel any possible outstanding interrupts if delays are enabled */ 2586 if (internals->link_up_delay_ms > 0 || 2587 internals->link_down_delay_ms > 0) 2588 rte_eal_alarm_cancel(bond_ethdev_delayed_lsc_propagation, 2589 bonded_eth_dev); 2590 2591 if (bonded_eth_dev->data->dev_link.link_status) { 2592 if (internals->link_up_delay_ms > 0) 2593 rte_eal_alarm_set(internals->link_up_delay_ms * 1000, 2594 bond_ethdev_delayed_lsc_propagation, 2595 (void *)bonded_eth_dev); 2596 else 2597 _rte_eth_dev_callback_process(bonded_eth_dev, 2598 RTE_ETH_EVENT_INTR_LSC, 2599 NULL, NULL); 2600 2601 } else { 2602 if (internals->link_down_delay_ms > 0) 2603 rte_eal_alarm_set(internals->link_down_delay_ms * 1000, 2604 bond_ethdev_delayed_lsc_propagation, 2605 (void *)bonded_eth_dev); 2606 else 2607 _rte_eth_dev_callback_process(bonded_eth_dev, 2608 RTE_ETH_EVENT_INTR_LSC, 2609 NULL, NULL); 2610 } 2611 } 2612 2613 rte_spinlock_unlock(&internals->lsc_lock); 2614 2615 return rc; 2616 } 2617 2618 static int 2619 bond_ethdev_rss_reta_update(struct rte_eth_dev *dev, 2620 struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size) 2621 { 2622 unsigned i, j; 2623 int result = 0; 2624 int slave_reta_size; 2625 unsigned reta_count; 2626 struct bond_dev_private *internals = dev->data->dev_private; 2627 2628 if (reta_size != internals->reta_size) 2629 return -EINVAL; 2630 2631 /* Copy RETA table */ 2632 reta_count = reta_size / RTE_RETA_GROUP_SIZE; 2633 2634 for (i = 0; i < reta_count; i++) { 2635 internals->reta_conf[i].mask = reta_conf[i].mask; 2636 for (j = 0; j < RTE_RETA_GROUP_SIZE; j++) 2637 if ((reta_conf[i].mask >> j) & 0x01) 2638 internals->reta_conf[i].reta[j] = reta_conf[i].reta[j]; 2639 } 2640 2641 /* Fill rest of array */ 2642 for (; i < RTE_DIM(internals->reta_conf); i += reta_count) 2643 memcpy(&internals->reta_conf[i], &internals->reta_conf[0], 2644 sizeof(internals->reta_conf[0]) * reta_count); 2645 2646 /* Propagate RETA over slaves */ 2647 for (i = 0; i < internals->slave_count; i++) { 2648 slave_reta_size = internals->slaves[i].reta_size; 2649 result = rte_eth_dev_rss_reta_update(internals->slaves[i].port_id, 2650 &internals->reta_conf[0], slave_reta_size); 2651 if (result < 0) 2652 return result; 2653 } 2654 2655 return 0; 2656 } 2657 2658 static int 2659 bond_ethdev_rss_reta_query(struct rte_eth_dev *dev, 2660 struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size) 2661 { 2662 int i, j; 2663 struct bond_dev_private *internals = dev->data->dev_private; 2664 2665 if (reta_size != internals->reta_size) 2666 return -EINVAL; 2667 2668 /* Copy RETA table */ 2669 for (i = 0; i < reta_size / RTE_RETA_GROUP_SIZE; i++) 2670 for (j = 0; j < RTE_RETA_GROUP_SIZE; j++) 2671 if ((reta_conf[i].mask >> j) & 0x01) 2672 reta_conf[i].reta[j] = internals->reta_conf[i].reta[j]; 2673 2674 return 0; 2675 } 2676 2677 static int 2678 bond_ethdev_rss_hash_update(struct rte_eth_dev *dev, 2679 struct rte_eth_rss_conf *rss_conf) 2680 { 2681 int i, result = 0; 2682 struct bond_dev_private *internals = dev->data->dev_private; 2683 struct rte_eth_rss_conf bond_rss_conf; 2684 2685 memcpy(&bond_rss_conf, rss_conf, sizeof(struct rte_eth_rss_conf)); 2686 2687 bond_rss_conf.rss_hf &= internals->flow_type_rss_offloads; 2688 2689 if (bond_rss_conf.rss_hf != 0) 2690 dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf = bond_rss_conf.rss_hf; 2691 2692 if (bond_rss_conf.rss_key && bond_rss_conf.rss_key_len < 2693 sizeof(internals->rss_key)) { 2694 if (bond_rss_conf.rss_key_len == 0) 2695 bond_rss_conf.rss_key_len = 40; 2696 internals->rss_key_len = bond_rss_conf.rss_key_len; 2697 memcpy(internals->rss_key, bond_rss_conf.rss_key, 2698 internals->rss_key_len); 2699 } 2700 2701 for (i = 0; i < internals->slave_count; i++) { 2702 result = rte_eth_dev_rss_hash_update(internals->slaves[i].port_id, 2703 &bond_rss_conf); 2704 if (result < 0) 2705 return result; 2706 } 2707 2708 return 0; 2709 } 2710 2711 static int 2712 bond_ethdev_rss_hash_conf_get(struct rte_eth_dev *dev, 2713 struct rte_eth_rss_conf *rss_conf) 2714 { 2715 struct bond_dev_private *internals = dev->data->dev_private; 2716 2717 rss_conf->rss_hf = dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf; 2718 rss_conf->rss_key_len = internals->rss_key_len; 2719 if (rss_conf->rss_key) 2720 memcpy(rss_conf->rss_key, internals->rss_key, internals->rss_key_len); 2721 2722 return 0; 2723 } 2724 2725 const struct eth_dev_ops default_dev_ops = { 2726 .dev_start = bond_ethdev_start, 2727 .dev_stop = bond_ethdev_stop, 2728 .dev_close = bond_ethdev_close, 2729 .dev_configure = bond_ethdev_configure, 2730 .dev_infos_get = bond_ethdev_info, 2731 .vlan_filter_set = bond_ethdev_vlan_filter_set, 2732 .rx_queue_setup = bond_ethdev_rx_queue_setup, 2733 .tx_queue_setup = bond_ethdev_tx_queue_setup, 2734 .rx_queue_release = bond_ethdev_rx_queue_release, 2735 .tx_queue_release = bond_ethdev_tx_queue_release, 2736 .link_update = bond_ethdev_link_update, 2737 .stats_get = bond_ethdev_stats_get, 2738 .stats_reset = bond_ethdev_stats_reset, 2739 .promiscuous_enable = bond_ethdev_promiscuous_enable, 2740 .promiscuous_disable = bond_ethdev_promiscuous_disable, 2741 .reta_update = bond_ethdev_rss_reta_update, 2742 .reta_query = bond_ethdev_rss_reta_query, 2743 .rss_hash_update = bond_ethdev_rss_hash_update, 2744 .rss_hash_conf_get = bond_ethdev_rss_hash_conf_get 2745 }; 2746 2747 static int 2748 bond_alloc(struct rte_vdev_device *dev, uint8_t mode) 2749 { 2750 const char *name = rte_vdev_device_name(dev); 2751 uint8_t socket_id = dev->device.numa_node; 2752 struct bond_dev_private *internals = NULL; 2753 struct rte_eth_dev *eth_dev = NULL; 2754 uint32_t vlan_filter_bmp_size; 2755 2756 /* now do all data allocation - for eth_dev structure, dummy pci driver 2757 * and internal (private) data 2758 */ 2759 2760 /* reserve an ethdev entry */ 2761 eth_dev = rte_eth_vdev_allocate(dev, sizeof(*internals)); 2762 if (eth_dev == NULL) { 2763 RTE_BOND_LOG(ERR, "Unable to allocate rte_eth_dev"); 2764 goto err; 2765 } 2766 2767 internals = eth_dev->data->dev_private; 2768 eth_dev->data->nb_rx_queues = (uint16_t)1; 2769 eth_dev->data->nb_tx_queues = (uint16_t)1; 2770 2771 eth_dev->data->mac_addrs = rte_zmalloc_socket(name, ETHER_ADDR_LEN, 0, 2772 socket_id); 2773 if (eth_dev->data->mac_addrs == NULL) { 2774 RTE_BOND_LOG(ERR, "Unable to malloc mac_addrs"); 2775 goto err; 2776 } 2777 2778 eth_dev->dev_ops = &default_dev_ops; 2779 eth_dev->data->dev_flags = RTE_ETH_DEV_INTR_LSC; 2780 2781 rte_spinlock_init(&internals->lock); 2782 rte_spinlock_init(&internals->lsc_lock); 2783 2784 internals->port_id = eth_dev->data->port_id; 2785 internals->mode = BONDING_MODE_INVALID; 2786 internals->current_primary_port = RTE_MAX_ETHPORTS + 1; 2787 internals->balance_xmit_policy = BALANCE_XMIT_POLICY_LAYER2; 2788 internals->xmit_hash = xmit_l2_hash; 2789 internals->user_defined_mac = 0; 2790 2791 internals->link_status_polling_enabled = 0; 2792 2793 internals->link_status_polling_interval_ms = 2794 DEFAULT_POLLING_INTERVAL_10_MS; 2795 internals->link_down_delay_ms = 0; 2796 internals->link_up_delay_ms = 0; 2797 2798 internals->slave_count = 0; 2799 internals->active_slave_count = 0; 2800 internals->rx_offload_capa = 0; 2801 internals->tx_offload_capa = 0; 2802 internals->candidate_max_rx_pktlen = 0; 2803 internals->max_rx_pktlen = 0; 2804 2805 /* Initially allow to choose any offload type */ 2806 internals->flow_type_rss_offloads = ETH_RSS_PROTO_MASK; 2807 2808 memset(internals->active_slaves, 0, sizeof(internals->active_slaves)); 2809 memset(internals->slaves, 0, sizeof(internals->slaves)); 2810 2811 /* Set mode 4 default configuration */ 2812 bond_mode_8023ad_setup(eth_dev, NULL); 2813 if (bond_ethdev_mode_set(eth_dev, mode)) { 2814 RTE_BOND_LOG(ERR, "Failed to set bonded device %d mode too %d", 2815 eth_dev->data->port_id, mode); 2816 goto err; 2817 } 2818 2819 vlan_filter_bmp_size = 2820 rte_bitmap_get_memory_footprint(ETHER_MAX_VLAN_ID + 1); 2821 internals->vlan_filter_bmpmem = rte_malloc(name, vlan_filter_bmp_size, 2822 RTE_CACHE_LINE_SIZE); 2823 if (internals->vlan_filter_bmpmem == NULL) { 2824 RTE_BOND_LOG(ERR, 2825 "Failed to allocate vlan bitmap for bonded device %u\n", 2826 eth_dev->data->port_id); 2827 goto err; 2828 } 2829 2830 internals->vlan_filter_bmp = rte_bitmap_init(ETHER_MAX_VLAN_ID + 1, 2831 internals->vlan_filter_bmpmem, vlan_filter_bmp_size); 2832 if (internals->vlan_filter_bmp == NULL) { 2833 RTE_BOND_LOG(ERR, 2834 "Failed to init vlan bitmap for bonded device %u\n", 2835 eth_dev->data->port_id); 2836 rte_free(internals->vlan_filter_bmpmem); 2837 goto err; 2838 } 2839 2840 return eth_dev->data->port_id; 2841 2842 err: 2843 rte_free(internals); 2844 if (eth_dev != NULL) { 2845 rte_free(eth_dev->data->mac_addrs); 2846 rte_eth_dev_release_port(eth_dev); 2847 } 2848 return -1; 2849 } 2850 2851 static int 2852 bond_probe(struct rte_vdev_device *dev) 2853 { 2854 const char *name; 2855 struct bond_dev_private *internals; 2856 struct rte_kvargs *kvlist; 2857 uint8_t bonding_mode, socket_id/*, agg_mode*/; 2858 int arg_count, port_id; 2859 uint8_t agg_mode; 2860 2861 if (!dev) 2862 return -EINVAL; 2863 2864 name = rte_vdev_device_name(dev); 2865 RTE_LOG(INFO, EAL, "Initializing pmd_bond for %s\n", name); 2866 2867 kvlist = rte_kvargs_parse(rte_vdev_device_args(dev), 2868 pmd_bond_init_valid_arguments); 2869 if (kvlist == NULL) 2870 return -1; 2871 2872 /* Parse link bonding mode */ 2873 if (rte_kvargs_count(kvlist, PMD_BOND_MODE_KVARG) == 1) { 2874 if (rte_kvargs_process(kvlist, PMD_BOND_MODE_KVARG, 2875 &bond_ethdev_parse_slave_mode_kvarg, 2876 &bonding_mode) != 0) { 2877 RTE_LOG(ERR, EAL, "Invalid mode for bonded device %s\n", 2878 name); 2879 goto parse_error; 2880 } 2881 } else { 2882 RTE_LOG(ERR, EAL, "Mode must be specified only once for bonded " 2883 "device %s\n", name); 2884 goto parse_error; 2885 } 2886 2887 /* Parse socket id to create bonding device on */ 2888 arg_count = rte_kvargs_count(kvlist, PMD_BOND_SOCKET_ID_KVARG); 2889 if (arg_count == 1) { 2890 if (rte_kvargs_process(kvlist, PMD_BOND_SOCKET_ID_KVARG, 2891 &bond_ethdev_parse_socket_id_kvarg, &socket_id) 2892 != 0) { 2893 RTE_LOG(ERR, EAL, "Invalid socket Id specified for " 2894 "bonded device %s\n", name); 2895 goto parse_error; 2896 } 2897 } else if (arg_count > 1) { 2898 RTE_LOG(ERR, EAL, "Socket Id can be specified only once for " 2899 "bonded device %s\n", name); 2900 goto parse_error; 2901 } else { 2902 socket_id = rte_socket_id(); 2903 } 2904 2905 dev->device.numa_node = socket_id; 2906 2907 /* Create link bonding eth device */ 2908 port_id = bond_alloc(dev, bonding_mode); 2909 if (port_id < 0) { 2910 RTE_LOG(ERR, EAL, "Failed to create socket %s in mode %u on " 2911 "socket %u.\n", name, bonding_mode, socket_id); 2912 goto parse_error; 2913 } 2914 internals = rte_eth_devices[port_id].data->dev_private; 2915 internals->kvlist = kvlist; 2916 2917 2918 if (rte_kvargs_count(kvlist, PMD_BOND_AGG_MODE_KVARG) == 1) { 2919 if (rte_kvargs_process(kvlist, 2920 PMD_BOND_AGG_MODE_KVARG, 2921 &bond_ethdev_parse_slave_agg_mode_kvarg, 2922 &agg_mode) != 0) { 2923 RTE_LOG(ERR, EAL, 2924 "Failed to parse agg selection mode for bonded device %s\n", 2925 name); 2926 goto parse_error; 2927 } 2928 2929 if (internals->mode == BONDING_MODE_8023AD) 2930 rte_eth_bond_8023ad_agg_selection_set(port_id, 2931 agg_mode); 2932 } else { 2933 rte_eth_bond_8023ad_agg_selection_set(port_id, AGG_STABLE); 2934 } 2935 2936 RTE_LOG(INFO, EAL, "Create bonded device %s on port %d in mode %u on " 2937 "socket %u.\n", name, port_id, bonding_mode, socket_id); 2938 return 0; 2939 2940 parse_error: 2941 rte_kvargs_free(kvlist); 2942 2943 return -1; 2944 } 2945 2946 static int 2947 bond_remove(struct rte_vdev_device *dev) 2948 { 2949 struct rte_eth_dev *eth_dev; 2950 struct bond_dev_private *internals; 2951 const char *name; 2952 2953 if (!dev) 2954 return -EINVAL; 2955 2956 name = rte_vdev_device_name(dev); 2957 RTE_LOG(INFO, EAL, "Uninitializing pmd_bond for %s\n", name); 2958 2959 /* now free all data allocation - for eth_dev structure, 2960 * dummy pci driver and internal (private) data 2961 */ 2962 2963 /* find an ethdev entry */ 2964 eth_dev = rte_eth_dev_allocated(name); 2965 if (eth_dev == NULL) 2966 return -ENODEV; 2967 2968 RTE_ASSERT(eth_dev->device == &dev->device); 2969 2970 internals = eth_dev->data->dev_private; 2971 if (internals->slave_count != 0) 2972 return -EBUSY; 2973 2974 if (eth_dev->data->dev_started == 1) { 2975 bond_ethdev_stop(eth_dev); 2976 bond_ethdev_close(eth_dev); 2977 } 2978 2979 eth_dev->dev_ops = NULL; 2980 eth_dev->rx_pkt_burst = NULL; 2981 eth_dev->tx_pkt_burst = NULL; 2982 2983 internals = eth_dev->data->dev_private; 2984 /* Try to release mempool used in mode6. If the bond 2985 * device is not mode6, free the NULL is not problem. 2986 */ 2987 rte_mempool_free(internals->mode6.mempool); 2988 rte_bitmap_free(internals->vlan_filter_bmp); 2989 rte_free(internals->vlan_filter_bmpmem); 2990 rte_free(eth_dev->data->dev_private); 2991 rte_free(eth_dev->data->mac_addrs); 2992 2993 rte_eth_dev_release_port(eth_dev); 2994 2995 return 0; 2996 } 2997 2998 /* this part will resolve the slave portids after all the other pdev and vdev 2999 * have been allocated */ 3000 static int 3001 bond_ethdev_configure(struct rte_eth_dev *dev) 3002 { 3003 const char *name = dev->device->name; 3004 struct bond_dev_private *internals = dev->data->dev_private; 3005 struct rte_kvargs *kvlist = internals->kvlist; 3006 int arg_count; 3007 uint16_t port_id = dev - rte_eth_devices; 3008 uint8_t agg_mode; 3009 3010 static const uint8_t default_rss_key[40] = { 3011 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2, 0x41, 0x67, 0x25, 0x3D, 3012 0x43, 0xA3, 0x8F, 0xB0, 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4, 3013 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C, 0x6A, 0x42, 0xB7, 0x3B, 3014 0xBE, 0xAC, 0x01, 0xFA 3015 }; 3016 3017 unsigned i, j; 3018 3019 /* If RSS is enabled, fill table and key with default values */ 3020 if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS) { 3021 dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key = internals->rss_key; 3022 dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len = 0; 3023 memcpy(internals->rss_key, default_rss_key, 40); 3024 3025 for (i = 0; i < RTE_DIM(internals->reta_conf); i++) { 3026 internals->reta_conf[i].mask = ~0LL; 3027 for (j = 0; j < RTE_RETA_GROUP_SIZE; j++) 3028 internals->reta_conf[i].reta[j] = j % dev->data->nb_rx_queues; 3029 } 3030 } 3031 3032 /* set the max_rx_pktlen */ 3033 internals->max_rx_pktlen = internals->candidate_max_rx_pktlen; 3034 3035 /* 3036 * if no kvlist, it means that this bonded device has been created 3037 * through the bonding api. 3038 */ 3039 if (!kvlist) 3040 return 0; 3041 3042 /* Parse MAC address for bonded device */ 3043 arg_count = rte_kvargs_count(kvlist, PMD_BOND_MAC_ADDR_KVARG); 3044 if (arg_count == 1) { 3045 struct ether_addr bond_mac; 3046 3047 if (rte_kvargs_process(kvlist, PMD_BOND_MAC_ADDR_KVARG, 3048 &bond_ethdev_parse_bond_mac_addr_kvarg, &bond_mac) < 0) { 3049 RTE_LOG(INFO, EAL, "Invalid mac address for bonded device %s\n", 3050 name); 3051 return -1; 3052 } 3053 3054 /* Set MAC address */ 3055 if (rte_eth_bond_mac_address_set(port_id, &bond_mac) != 0) { 3056 RTE_LOG(ERR, EAL, 3057 "Failed to set mac address on bonded device %s\n", 3058 name); 3059 return -1; 3060 } 3061 } else if (arg_count > 1) { 3062 RTE_LOG(ERR, EAL, 3063 "MAC address can be specified only once for bonded device %s\n", 3064 name); 3065 return -1; 3066 } 3067 3068 /* Parse/set balance mode transmit policy */ 3069 arg_count = rte_kvargs_count(kvlist, PMD_BOND_XMIT_POLICY_KVARG); 3070 if (arg_count == 1) { 3071 uint8_t xmit_policy; 3072 3073 if (rte_kvargs_process(kvlist, PMD_BOND_XMIT_POLICY_KVARG, 3074 &bond_ethdev_parse_balance_xmit_policy_kvarg, &xmit_policy) != 3075 0) { 3076 RTE_LOG(INFO, EAL, 3077 "Invalid xmit policy specified for bonded device %s\n", 3078 name); 3079 return -1; 3080 } 3081 3082 /* Set balance mode transmit policy*/ 3083 if (rte_eth_bond_xmit_policy_set(port_id, xmit_policy) != 0) { 3084 RTE_LOG(ERR, EAL, 3085 "Failed to set balance xmit policy on bonded device %s\n", 3086 name); 3087 return -1; 3088 } 3089 } else if (arg_count > 1) { 3090 RTE_LOG(ERR, EAL, 3091 "Transmit policy can be specified only once for bonded device" 3092 " %s\n", name); 3093 return -1; 3094 } 3095 3096 if (rte_kvargs_count(kvlist, PMD_BOND_AGG_MODE_KVARG) == 1) { 3097 if (rte_kvargs_process(kvlist, 3098 PMD_BOND_AGG_MODE_KVARG, 3099 &bond_ethdev_parse_slave_agg_mode_kvarg, 3100 &agg_mode) != 0) { 3101 RTE_LOG(ERR, EAL, 3102 "Failed to parse agg selection mode for bonded device %s\n", 3103 name); 3104 } 3105 if (internals->mode == BONDING_MODE_8023AD) 3106 rte_eth_bond_8023ad_agg_selection_set(port_id, 3107 agg_mode); 3108 } 3109 3110 /* Parse/add slave ports to bonded device */ 3111 if (rte_kvargs_count(kvlist, PMD_BOND_SLAVE_PORT_KVARG) > 0) { 3112 struct bond_ethdev_slave_ports slave_ports; 3113 unsigned i; 3114 3115 memset(&slave_ports, 0, sizeof(slave_ports)); 3116 3117 if (rte_kvargs_process(kvlist, PMD_BOND_SLAVE_PORT_KVARG, 3118 &bond_ethdev_parse_slave_port_kvarg, &slave_ports) != 0) { 3119 RTE_LOG(ERR, EAL, 3120 "Failed to parse slave ports for bonded device %s\n", 3121 name); 3122 return -1; 3123 } 3124 3125 for (i = 0; i < slave_ports.slave_count; i++) { 3126 if (rte_eth_bond_slave_add(port_id, slave_ports.slaves[i]) != 0) { 3127 RTE_LOG(ERR, EAL, 3128 "Failed to add port %d as slave to bonded device %s\n", 3129 slave_ports.slaves[i], name); 3130 } 3131 } 3132 3133 } else { 3134 RTE_LOG(INFO, EAL, "No slaves specified for bonded device %s\n", name); 3135 return -1; 3136 } 3137 3138 /* Parse/set primary slave port id*/ 3139 arg_count = rte_kvargs_count(kvlist, PMD_BOND_PRIMARY_SLAVE_KVARG); 3140 if (arg_count == 1) { 3141 uint16_t primary_slave_port_id; 3142 3143 if (rte_kvargs_process(kvlist, 3144 PMD_BOND_PRIMARY_SLAVE_KVARG, 3145 &bond_ethdev_parse_primary_slave_port_id_kvarg, 3146 &primary_slave_port_id) < 0) { 3147 RTE_LOG(INFO, EAL, 3148 "Invalid primary slave port id specified for bonded device" 3149 " %s\n", name); 3150 return -1; 3151 } 3152 3153 /* Set balance mode transmit policy*/ 3154 if (rte_eth_bond_primary_set(port_id, primary_slave_port_id) 3155 != 0) { 3156 RTE_LOG(ERR, EAL, 3157 "Failed to set primary slave port %d on bonded device %s\n", 3158 primary_slave_port_id, name); 3159 return -1; 3160 } 3161 } else if (arg_count > 1) { 3162 RTE_LOG(INFO, EAL, 3163 "Primary slave can be specified only once for bonded device" 3164 " %s\n", name); 3165 return -1; 3166 } 3167 3168 /* Parse link status monitor polling interval */ 3169 arg_count = rte_kvargs_count(kvlist, PMD_BOND_LSC_POLL_PERIOD_KVARG); 3170 if (arg_count == 1) { 3171 uint32_t lsc_poll_interval_ms; 3172 3173 if (rte_kvargs_process(kvlist, 3174 PMD_BOND_LSC_POLL_PERIOD_KVARG, 3175 &bond_ethdev_parse_time_ms_kvarg, 3176 &lsc_poll_interval_ms) < 0) { 3177 RTE_LOG(INFO, EAL, 3178 "Invalid lsc polling interval value specified for bonded" 3179 " device %s\n", name); 3180 return -1; 3181 } 3182 3183 if (rte_eth_bond_link_monitoring_set(port_id, lsc_poll_interval_ms) 3184 != 0) { 3185 RTE_LOG(ERR, EAL, 3186 "Failed to set lsc monitor polling interval (%u ms) on" 3187 " bonded device %s\n", lsc_poll_interval_ms, name); 3188 return -1; 3189 } 3190 } else if (arg_count > 1) { 3191 RTE_LOG(INFO, EAL, 3192 "LSC polling interval can be specified only once for bonded" 3193 " device %s\n", name); 3194 return -1; 3195 } 3196 3197 /* Parse link up interrupt propagation delay */ 3198 arg_count = rte_kvargs_count(kvlist, PMD_BOND_LINK_UP_PROP_DELAY_KVARG); 3199 if (arg_count == 1) { 3200 uint32_t link_up_delay_ms; 3201 3202 if (rte_kvargs_process(kvlist, 3203 PMD_BOND_LINK_UP_PROP_DELAY_KVARG, 3204 &bond_ethdev_parse_time_ms_kvarg, 3205 &link_up_delay_ms) < 0) { 3206 RTE_LOG(INFO, EAL, 3207 "Invalid link up propagation delay value specified for" 3208 " bonded device %s\n", name); 3209 return -1; 3210 } 3211 3212 /* Set balance mode transmit policy*/ 3213 if (rte_eth_bond_link_up_prop_delay_set(port_id, link_up_delay_ms) 3214 != 0) { 3215 RTE_LOG(ERR, EAL, 3216 "Failed to set link up propagation delay (%u ms) on bonded" 3217 " device %s\n", link_up_delay_ms, name); 3218 return -1; 3219 } 3220 } else if (arg_count > 1) { 3221 RTE_LOG(INFO, EAL, 3222 "Link up propagation delay can be specified only once for" 3223 " bonded device %s\n", name); 3224 return -1; 3225 } 3226 3227 /* Parse link down interrupt propagation delay */ 3228 arg_count = rte_kvargs_count(kvlist, PMD_BOND_LINK_DOWN_PROP_DELAY_KVARG); 3229 if (arg_count == 1) { 3230 uint32_t link_down_delay_ms; 3231 3232 if (rte_kvargs_process(kvlist, 3233 PMD_BOND_LINK_DOWN_PROP_DELAY_KVARG, 3234 &bond_ethdev_parse_time_ms_kvarg, 3235 &link_down_delay_ms) < 0) { 3236 RTE_LOG(INFO, EAL, 3237 "Invalid link down propagation delay value specified for" 3238 " bonded device %s\n", name); 3239 return -1; 3240 } 3241 3242 /* Set balance mode transmit policy*/ 3243 if (rte_eth_bond_link_down_prop_delay_set(port_id, link_down_delay_ms) 3244 != 0) { 3245 RTE_LOG(ERR, EAL, 3246 "Failed to set link down propagation delay (%u ms) on" 3247 " bonded device %s\n", link_down_delay_ms, name); 3248 return -1; 3249 } 3250 } else if (arg_count > 1) { 3251 RTE_LOG(INFO, EAL, 3252 "Link down propagation delay can be specified only once for" 3253 " bonded device %s\n", name); 3254 return -1; 3255 } 3256 3257 return 0; 3258 } 3259 3260 struct rte_vdev_driver pmd_bond_drv = { 3261 .probe = bond_probe, 3262 .remove = bond_remove, 3263 }; 3264 3265 RTE_PMD_REGISTER_VDEV(net_bonding, pmd_bond_drv); 3266 RTE_PMD_REGISTER_ALIAS(net_bonding, eth_bond); 3267 3268 RTE_PMD_REGISTER_PARAM_STRING(net_bonding, 3269 "slave=<ifc> " 3270 "primary=<ifc> " 3271 "mode=[0-6] " 3272 "xmit_policy=[l2 | l23 | l34] " 3273 "agg_mode=[count | stable | bandwidth] " 3274 "socket_id=<int> " 3275 "mac=<mac addr> " 3276 "lsc_poll_period_ms=<int> " 3277 "up_delay=<int> " 3278 "down_delay=<int>"); 3279