1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <rte_atomic.h> 35 #include <rte_branch_prediction.h> 36 #include <rte_byteorder.h> 37 #include <rte_common.h> 38 #include <rte_mbuf.h> 39 #include <rte_ethdev.h> 40 #include <rte_ethdev_vdev.h> 41 #include <rte_malloc.h> 42 #include <rte_bus_vdev.h> 43 #include <rte_kvargs.h> 44 #include <rte_net.h> 45 #include <rte_debug.h> 46 #include <rte_ip.h> 47 48 #include <sys/types.h> 49 #include <sys/stat.h> 50 #include <sys/socket.h> 51 #include <sys/ioctl.h> 52 #include <sys/utsname.h> 53 #include <sys/mman.h> 54 #include <errno.h> 55 #include <signal.h> 56 #include <stdint.h> 57 #include <sys/uio.h> 58 #include <unistd.h> 59 #include <arpa/inet.h> 60 #include <net/if.h> 61 #include <linux/if_tun.h> 62 #include <linux/if_ether.h> 63 #include <fcntl.h> 64 65 #include <rte_eth_tap.h> 66 #include <tap_flow.h> 67 #include <tap_netlink.h> 68 #include <tap_tcmsgs.h> 69 70 /* Linux based path to the TUN device */ 71 #define TUN_TAP_DEV_PATH "/dev/net/tun" 72 #define DEFAULT_TAP_NAME "dtap" 73 74 #define ETH_TAP_IFACE_ARG "iface" 75 #define ETH_TAP_SPEED_ARG "speed" 76 #define ETH_TAP_REMOTE_ARG "remote" 77 #define ETH_TAP_MAC_ARG "mac" 78 #define ETH_TAP_MAC_FIXED "fixed" 79 80 static struct rte_vdev_driver pmd_tap_drv; 81 82 static const char *valid_arguments[] = { 83 ETH_TAP_IFACE_ARG, 84 ETH_TAP_SPEED_ARG, 85 ETH_TAP_REMOTE_ARG, 86 ETH_TAP_MAC_ARG, 87 NULL 88 }; 89 90 static int tap_unit; 91 92 static volatile uint32_t tap_trigger; /* Rx trigger */ 93 94 static struct rte_eth_link pmd_link = { 95 .link_speed = ETH_SPEED_NUM_10G, 96 .link_duplex = ETH_LINK_FULL_DUPLEX, 97 .link_status = ETH_LINK_DOWN, 98 .link_autoneg = ETH_LINK_AUTONEG 99 }; 100 101 static void 102 tap_trigger_cb(int sig __rte_unused) 103 { 104 /* Valid trigger values are nonzero */ 105 tap_trigger = (tap_trigger + 1) | 0x80000000; 106 } 107 108 /* Specifies on what netdevices the ioctl should be applied */ 109 enum ioctl_mode { 110 LOCAL_AND_REMOTE, 111 LOCAL_ONLY, 112 REMOTE_ONLY, 113 }; 114 115 static int tap_intr_handle_set(struct rte_eth_dev *dev, int set); 116 117 /* Tun/Tap allocation routine 118 * 119 * name is the number of the interface to use, unless NULL to take the host 120 * supplied name. 121 */ 122 static int 123 tun_alloc(struct pmd_internals *pmd) 124 { 125 struct ifreq ifr; 126 #ifdef IFF_MULTI_QUEUE 127 unsigned int features; 128 #endif 129 int fd; 130 131 memset(&ifr, 0, sizeof(struct ifreq)); 132 133 /* 134 * Do not set IFF_NO_PI as packet information header will be needed 135 * to check if a received packet has been truncated. 136 */ 137 ifr.ifr_flags = IFF_TAP; 138 snprintf(ifr.ifr_name, IFNAMSIZ, "%s", pmd->name); 139 140 RTE_LOG(DEBUG, PMD, "ifr_name '%s'\n", ifr.ifr_name); 141 142 fd = open(TUN_TAP_DEV_PATH, O_RDWR); 143 if (fd < 0) { 144 RTE_LOG(ERR, PMD, "Unable to create TAP interface\n"); 145 goto error; 146 } 147 148 #ifdef IFF_MULTI_QUEUE 149 /* Grab the TUN features to verify we can work multi-queue */ 150 if (ioctl(fd, TUNGETFEATURES, &features) < 0) { 151 RTE_LOG(ERR, PMD, "TAP unable to get TUN/TAP features\n"); 152 goto error; 153 } 154 RTE_LOG(DEBUG, PMD, " TAP Features %08x\n", features); 155 156 if (features & IFF_MULTI_QUEUE) { 157 RTE_LOG(DEBUG, PMD, " Multi-queue support for %d queues\n", 158 RTE_PMD_TAP_MAX_QUEUES); 159 ifr.ifr_flags |= IFF_MULTI_QUEUE; 160 } else 161 #endif 162 { 163 ifr.ifr_flags |= IFF_ONE_QUEUE; 164 RTE_LOG(DEBUG, PMD, " Single queue only support\n"); 165 } 166 167 /* Set the TUN/TAP configuration and set the name if needed */ 168 if (ioctl(fd, TUNSETIFF, (void *)&ifr) < 0) { 169 RTE_LOG(WARNING, PMD, 170 "Unable to set TUNSETIFF for %s\n", 171 ifr.ifr_name); 172 perror("TUNSETIFF"); 173 goto error; 174 } 175 176 /* Always set the file descriptor to non-blocking */ 177 if (fcntl(fd, F_SETFL, O_NONBLOCK) < 0) { 178 RTE_LOG(WARNING, PMD, 179 "Unable to set %s to nonblocking\n", 180 ifr.ifr_name); 181 perror("F_SETFL, NONBLOCK"); 182 goto error; 183 } 184 185 /* Set up trigger to optimize empty Rx bursts */ 186 errno = 0; 187 do { 188 struct sigaction sa; 189 int flags = fcntl(fd, F_GETFL); 190 191 if (flags == -1 || sigaction(SIGIO, NULL, &sa) == -1) 192 break; 193 if (sa.sa_handler != tap_trigger_cb) { 194 /* 195 * Make sure SIGIO is not already taken. This is done 196 * as late as possible to leave the application a 197 * chance to set up its own signal handler first. 198 */ 199 if (sa.sa_handler != SIG_IGN && 200 sa.sa_handler != SIG_DFL) { 201 errno = EBUSY; 202 break; 203 } 204 sa = (struct sigaction){ 205 .sa_flags = SA_RESTART, 206 .sa_handler = tap_trigger_cb, 207 }; 208 if (sigaction(SIGIO, &sa, NULL) == -1) 209 break; 210 } 211 /* Enable SIGIO on file descriptor */ 212 fcntl(fd, F_SETFL, flags | O_ASYNC); 213 fcntl(fd, F_SETOWN, getpid()); 214 } while (0); 215 if (errno) { 216 /* Disable trigger globally in case of error */ 217 tap_trigger = 0; 218 RTE_LOG(WARNING, PMD, "Rx trigger disabled: %s\n", 219 strerror(errno)); 220 } 221 222 return fd; 223 224 error: 225 if (fd > 0) 226 close(fd); 227 return -1; 228 } 229 230 static void 231 tap_verify_csum(struct rte_mbuf *mbuf) 232 { 233 uint32_t l2 = mbuf->packet_type & RTE_PTYPE_L2_MASK; 234 uint32_t l3 = mbuf->packet_type & RTE_PTYPE_L3_MASK; 235 uint32_t l4 = mbuf->packet_type & RTE_PTYPE_L4_MASK; 236 unsigned int l2_len = sizeof(struct ether_hdr); 237 unsigned int l3_len; 238 uint16_t cksum = 0; 239 void *l3_hdr; 240 void *l4_hdr; 241 242 if (l2 == RTE_PTYPE_L2_ETHER_VLAN) 243 l2_len += 4; 244 else if (l2 == RTE_PTYPE_L2_ETHER_QINQ) 245 l2_len += 8; 246 /* Don't verify checksum for packets with discontinuous L2 header */ 247 if (unlikely(l2_len + sizeof(struct ipv4_hdr) > 248 rte_pktmbuf_data_len(mbuf))) 249 return; 250 l3_hdr = rte_pktmbuf_mtod_offset(mbuf, void *, l2_len); 251 if (l3 == RTE_PTYPE_L3_IPV4 || l3 == RTE_PTYPE_L3_IPV4_EXT) { 252 struct ipv4_hdr *iph = l3_hdr; 253 254 /* ihl contains the number of 4-byte words in the header */ 255 l3_len = 4 * (iph->version_ihl & 0xf); 256 if (unlikely(l2_len + l3_len > rte_pktmbuf_data_len(mbuf))) 257 return; 258 259 cksum = ~rte_raw_cksum(iph, l3_len); 260 mbuf->ol_flags |= cksum ? 261 PKT_RX_IP_CKSUM_BAD : 262 PKT_RX_IP_CKSUM_GOOD; 263 } else if (l3 == RTE_PTYPE_L3_IPV6) { 264 l3_len = sizeof(struct ipv6_hdr); 265 } else { 266 /* IPv6 extensions are not supported */ 267 return; 268 } 269 if (l4 == RTE_PTYPE_L4_UDP || l4 == RTE_PTYPE_L4_TCP) { 270 l4_hdr = rte_pktmbuf_mtod_offset(mbuf, void *, l2_len + l3_len); 271 /* Don't verify checksum for multi-segment packets. */ 272 if (mbuf->nb_segs > 1) 273 return; 274 if (l3 == RTE_PTYPE_L3_IPV4) 275 cksum = ~rte_ipv4_udptcp_cksum(l3_hdr, l4_hdr); 276 else if (l3 == RTE_PTYPE_L3_IPV6) 277 cksum = ~rte_ipv6_udptcp_cksum(l3_hdr, l4_hdr); 278 mbuf->ol_flags |= cksum ? 279 PKT_RX_L4_CKSUM_BAD : 280 PKT_RX_L4_CKSUM_GOOD; 281 } 282 } 283 284 /* Callback to handle the rx burst of packets to the correct interface and 285 * file descriptor(s) in a multi-queue setup. 286 */ 287 static uint16_t 288 pmd_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) 289 { 290 struct rx_queue *rxq = queue; 291 uint16_t num_rx; 292 unsigned long num_rx_bytes = 0; 293 uint32_t trigger = tap_trigger; 294 295 if (trigger == rxq->trigger_seen) 296 return 0; 297 if (trigger) 298 rxq->trigger_seen = trigger; 299 rte_compiler_barrier(); 300 for (num_rx = 0; num_rx < nb_pkts; ) { 301 struct rte_mbuf *mbuf = rxq->pool; 302 struct rte_mbuf *seg = NULL; 303 struct rte_mbuf *new_tail = NULL; 304 uint16_t data_off = rte_pktmbuf_headroom(mbuf); 305 int len; 306 307 len = readv(rxq->fd, *rxq->iovecs, 308 1 + (rxq->rxmode->enable_scatter ? 309 rxq->nb_rx_desc : 1)); 310 if (len < (int)sizeof(struct tun_pi)) 311 break; 312 313 /* Packet couldn't fit in the provided mbuf */ 314 if (unlikely(rxq->pi.flags & TUN_PKT_STRIP)) { 315 rxq->stats.ierrors++; 316 continue; 317 } 318 319 len -= sizeof(struct tun_pi); 320 321 mbuf->pkt_len = len; 322 mbuf->port = rxq->in_port; 323 while (1) { 324 struct rte_mbuf *buf = rte_pktmbuf_alloc(rxq->mp); 325 326 if (unlikely(!buf)) { 327 rxq->stats.rx_nombuf++; 328 /* No new buf has been allocated: do nothing */ 329 if (!new_tail || !seg) 330 goto end; 331 332 seg->next = NULL; 333 rte_pktmbuf_free(mbuf); 334 335 goto end; 336 } 337 seg = seg ? seg->next : mbuf; 338 if (rxq->pool == mbuf) 339 rxq->pool = buf; 340 if (new_tail) 341 new_tail->next = buf; 342 new_tail = buf; 343 new_tail->next = seg->next; 344 345 /* iovecs[0] is reserved for packet info (pi) */ 346 (*rxq->iovecs)[mbuf->nb_segs].iov_len = 347 buf->buf_len - data_off; 348 (*rxq->iovecs)[mbuf->nb_segs].iov_base = 349 (char *)buf->buf_addr + data_off; 350 351 seg->data_len = RTE_MIN(seg->buf_len - data_off, len); 352 seg->data_off = data_off; 353 354 len -= seg->data_len; 355 if (len <= 0) 356 break; 357 mbuf->nb_segs++; 358 /* First segment has headroom, not the others */ 359 data_off = 0; 360 } 361 seg->next = NULL; 362 mbuf->packet_type = rte_net_get_ptype(mbuf, NULL, 363 RTE_PTYPE_ALL_MASK); 364 if (rxq->rxmode->hw_ip_checksum) 365 tap_verify_csum(mbuf); 366 367 /* account for the receive frame */ 368 bufs[num_rx++] = mbuf; 369 num_rx_bytes += mbuf->pkt_len; 370 } 371 end: 372 rxq->stats.ipackets += num_rx; 373 rxq->stats.ibytes += num_rx_bytes; 374 375 return num_rx; 376 } 377 378 static void 379 tap_tx_offload(char *packet, uint64_t ol_flags, unsigned int l2_len, 380 unsigned int l3_len) 381 { 382 void *l3_hdr = packet + l2_len; 383 384 if (ol_flags & (PKT_TX_IP_CKSUM | PKT_TX_IPV4)) { 385 struct ipv4_hdr *iph = l3_hdr; 386 uint16_t cksum; 387 388 iph->hdr_checksum = 0; 389 cksum = rte_raw_cksum(iph, l3_len); 390 iph->hdr_checksum = (cksum == 0xffff) ? cksum : ~cksum; 391 } 392 if (ol_flags & PKT_TX_L4_MASK) { 393 uint16_t l4_len; 394 uint32_t cksum; 395 uint16_t *l4_cksum; 396 void *l4_hdr; 397 398 l4_hdr = packet + l2_len + l3_len; 399 if ((ol_flags & PKT_TX_L4_MASK) == PKT_TX_UDP_CKSUM) 400 l4_cksum = &((struct udp_hdr *)l4_hdr)->dgram_cksum; 401 else if ((ol_flags & PKT_TX_L4_MASK) == PKT_TX_TCP_CKSUM) 402 l4_cksum = &((struct tcp_hdr *)l4_hdr)->cksum; 403 else 404 return; 405 *l4_cksum = 0; 406 if (ol_flags & PKT_TX_IPV4) { 407 struct ipv4_hdr *iph = l3_hdr; 408 409 l4_len = rte_be_to_cpu_16(iph->total_length) - l3_len; 410 cksum = rte_ipv4_phdr_cksum(l3_hdr, 0); 411 } else { 412 struct ipv6_hdr *ip6h = l3_hdr; 413 414 /* payload_len does not include ext headers */ 415 l4_len = rte_be_to_cpu_16(ip6h->payload_len) - 416 l3_len + sizeof(struct ipv6_hdr); 417 cksum = rte_ipv6_phdr_cksum(l3_hdr, 0); 418 } 419 cksum += rte_raw_cksum(l4_hdr, l4_len); 420 cksum = ((cksum & 0xffff0000) >> 16) + (cksum & 0xffff); 421 cksum = (~cksum) & 0xffff; 422 if (cksum == 0) 423 cksum = 0xffff; 424 *l4_cksum = cksum; 425 } 426 } 427 428 /* Callback to handle sending packets from the tap interface 429 */ 430 static uint16_t 431 pmd_tx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) 432 { 433 struct tx_queue *txq = queue; 434 uint16_t num_tx = 0; 435 unsigned long num_tx_bytes = 0; 436 uint32_t max_size; 437 int i; 438 439 if (unlikely(nb_pkts == 0)) 440 return 0; 441 442 max_size = *txq->mtu + (ETHER_HDR_LEN + ETHER_CRC_LEN + 4); 443 for (i = 0; i < nb_pkts; i++) { 444 struct rte_mbuf *mbuf = bufs[num_tx]; 445 struct iovec iovecs[mbuf->nb_segs + 1]; 446 struct tun_pi pi = { .flags = 0 }; 447 struct rte_mbuf *seg = mbuf; 448 char m_copy[mbuf->data_len]; 449 int n; 450 int j; 451 452 /* stats.errs will be incremented */ 453 if (rte_pktmbuf_pkt_len(mbuf) > max_size) 454 break; 455 456 iovecs[0].iov_base = π 457 iovecs[0].iov_len = sizeof(pi); 458 for (j = 1; j <= mbuf->nb_segs; j++) { 459 iovecs[j].iov_len = rte_pktmbuf_data_len(seg); 460 iovecs[j].iov_base = 461 rte_pktmbuf_mtod(seg, void *); 462 seg = seg->next; 463 } 464 if (mbuf->ol_flags & (PKT_TX_IP_CKSUM | PKT_TX_IPV4) || 465 (mbuf->ol_flags & PKT_TX_L4_MASK) == PKT_TX_UDP_CKSUM || 466 (mbuf->ol_flags & PKT_TX_L4_MASK) == PKT_TX_TCP_CKSUM) { 467 /* Support only packets with all data in the same seg */ 468 if (mbuf->nb_segs > 1) 469 break; 470 /* To change checksums, work on a copy of data. */ 471 rte_memcpy(m_copy, rte_pktmbuf_mtod(mbuf, void *), 472 rte_pktmbuf_data_len(mbuf)); 473 tap_tx_offload(m_copy, mbuf->ol_flags, 474 mbuf->l2_len, mbuf->l3_len); 475 iovecs[1].iov_base = m_copy; 476 } 477 /* copy the tx frame data */ 478 n = writev(txq->fd, iovecs, mbuf->nb_segs + 1); 479 if (n <= 0) 480 break; 481 482 num_tx++; 483 num_tx_bytes += mbuf->pkt_len; 484 rte_pktmbuf_free(mbuf); 485 } 486 487 txq->stats.opackets += num_tx; 488 txq->stats.errs += nb_pkts - num_tx; 489 txq->stats.obytes += num_tx_bytes; 490 491 return num_tx; 492 } 493 494 static const char * 495 tap_ioctl_req2str(unsigned long request) 496 { 497 switch (request) { 498 case SIOCSIFFLAGS: 499 return "SIOCSIFFLAGS"; 500 case SIOCGIFFLAGS: 501 return "SIOCGIFFLAGS"; 502 case SIOCGIFHWADDR: 503 return "SIOCGIFHWADDR"; 504 case SIOCSIFHWADDR: 505 return "SIOCSIFHWADDR"; 506 case SIOCSIFMTU: 507 return "SIOCSIFMTU"; 508 } 509 return "UNKNOWN"; 510 } 511 512 static int 513 tap_ioctl(struct pmd_internals *pmd, unsigned long request, 514 struct ifreq *ifr, int set, enum ioctl_mode mode) 515 { 516 short req_flags = ifr->ifr_flags; 517 int remote = pmd->remote_if_index && 518 (mode == REMOTE_ONLY || mode == LOCAL_AND_REMOTE); 519 520 if (!pmd->remote_if_index && mode == REMOTE_ONLY) 521 return 0; 522 /* 523 * If there is a remote netdevice, apply ioctl on it, then apply it on 524 * the tap netdevice. 525 */ 526 apply: 527 if (remote) 528 snprintf(ifr->ifr_name, IFNAMSIZ, "%s", pmd->remote_iface); 529 else if (mode == LOCAL_ONLY || mode == LOCAL_AND_REMOTE) 530 snprintf(ifr->ifr_name, IFNAMSIZ, "%s", pmd->name); 531 switch (request) { 532 case SIOCSIFFLAGS: 533 /* fetch current flags to leave other flags untouched */ 534 if (ioctl(pmd->ioctl_sock, SIOCGIFFLAGS, ifr) < 0) 535 goto error; 536 if (set) 537 ifr->ifr_flags |= req_flags; 538 else 539 ifr->ifr_flags &= ~req_flags; 540 break; 541 case SIOCGIFFLAGS: 542 case SIOCGIFHWADDR: 543 case SIOCSIFHWADDR: 544 case SIOCSIFMTU: 545 break; 546 default: 547 RTE_ASSERT(!"unsupported request type: must not happen"); 548 } 549 if (ioctl(pmd->ioctl_sock, request, ifr) < 0) 550 goto error; 551 if (remote-- && mode == LOCAL_AND_REMOTE) 552 goto apply; 553 return 0; 554 555 error: 556 RTE_LOG(DEBUG, PMD, "%s: %s(%s) failed: %s(%d)\n", ifr->ifr_name, 557 __func__, tap_ioctl_req2str(request), strerror(errno), errno); 558 return -errno; 559 } 560 561 static int 562 tap_link_set_down(struct rte_eth_dev *dev) 563 { 564 struct pmd_internals *pmd = dev->data->dev_private; 565 struct ifreq ifr = { .ifr_flags = IFF_UP }; 566 567 dev->data->dev_link.link_status = ETH_LINK_DOWN; 568 return tap_ioctl(pmd, SIOCSIFFLAGS, &ifr, 0, LOCAL_ONLY); 569 } 570 571 static int 572 tap_link_set_up(struct rte_eth_dev *dev) 573 { 574 struct pmd_internals *pmd = dev->data->dev_private; 575 struct ifreq ifr = { .ifr_flags = IFF_UP }; 576 577 dev->data->dev_link.link_status = ETH_LINK_UP; 578 return tap_ioctl(pmd, SIOCSIFFLAGS, &ifr, 1, LOCAL_AND_REMOTE); 579 } 580 581 static int 582 tap_dev_start(struct rte_eth_dev *dev) 583 { 584 int err; 585 586 err = tap_intr_handle_set(dev, 1); 587 if (err) 588 return err; 589 return tap_link_set_up(dev); 590 } 591 592 /* This function gets called when the current port gets stopped. 593 */ 594 static void 595 tap_dev_stop(struct rte_eth_dev *dev) 596 { 597 tap_intr_handle_set(dev, 0); 598 tap_link_set_down(dev); 599 } 600 601 static int 602 tap_dev_configure(struct rte_eth_dev *dev) 603 { 604 if (dev->data->nb_rx_queues > RTE_PMD_TAP_MAX_QUEUES) { 605 RTE_LOG(ERR, PMD, 606 "%s: number of rx queues %d exceeds max num of queues %d\n", 607 dev->device->name, 608 dev->data->nb_rx_queues, 609 RTE_PMD_TAP_MAX_QUEUES); 610 return -1; 611 } 612 if (dev->data->nb_tx_queues > RTE_PMD_TAP_MAX_QUEUES) { 613 RTE_LOG(ERR, PMD, 614 "%s: number of tx queues %d exceeds max num of queues %d\n", 615 dev->device->name, 616 dev->data->nb_tx_queues, 617 RTE_PMD_TAP_MAX_QUEUES); 618 return -1; 619 } 620 621 RTE_LOG(INFO, PMD, "%s: %p: TX configured queues number: %u\n", 622 dev->device->name, (void *)dev, dev->data->nb_tx_queues); 623 624 RTE_LOG(INFO, PMD, "%s: %p: RX configured queues number: %u\n", 625 dev->device->name, (void *)dev, dev->data->nb_rx_queues); 626 627 return 0; 628 } 629 630 static uint32_t 631 tap_dev_speed_capa(void) 632 { 633 uint32_t speed = pmd_link.link_speed; 634 uint32_t capa = 0; 635 636 if (speed >= ETH_SPEED_NUM_10M) 637 capa |= ETH_LINK_SPEED_10M; 638 if (speed >= ETH_SPEED_NUM_100M) 639 capa |= ETH_LINK_SPEED_100M; 640 if (speed >= ETH_SPEED_NUM_1G) 641 capa |= ETH_LINK_SPEED_1G; 642 if (speed >= ETH_SPEED_NUM_5G) 643 capa |= ETH_LINK_SPEED_2_5G; 644 if (speed >= ETH_SPEED_NUM_5G) 645 capa |= ETH_LINK_SPEED_5G; 646 if (speed >= ETH_SPEED_NUM_10G) 647 capa |= ETH_LINK_SPEED_10G; 648 if (speed >= ETH_SPEED_NUM_20G) 649 capa |= ETH_LINK_SPEED_20G; 650 if (speed >= ETH_SPEED_NUM_25G) 651 capa |= ETH_LINK_SPEED_25G; 652 if (speed >= ETH_SPEED_NUM_40G) 653 capa |= ETH_LINK_SPEED_40G; 654 if (speed >= ETH_SPEED_NUM_50G) 655 capa |= ETH_LINK_SPEED_50G; 656 if (speed >= ETH_SPEED_NUM_56G) 657 capa |= ETH_LINK_SPEED_56G; 658 if (speed >= ETH_SPEED_NUM_100G) 659 capa |= ETH_LINK_SPEED_100G; 660 661 return capa; 662 } 663 664 static void 665 tap_dev_info(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 666 { 667 struct pmd_internals *internals = dev->data->dev_private; 668 669 dev_info->if_index = internals->if_index; 670 dev_info->max_mac_addrs = 1; 671 dev_info->max_rx_pktlen = (uint32_t)ETHER_MAX_VLAN_FRAME_LEN; 672 dev_info->max_rx_queues = RTE_PMD_TAP_MAX_QUEUES; 673 dev_info->max_tx_queues = RTE_PMD_TAP_MAX_QUEUES; 674 dev_info->min_rx_bufsize = 0; 675 dev_info->pci_dev = NULL; 676 dev_info->speed_capa = tap_dev_speed_capa(); 677 dev_info->rx_offload_capa = (DEV_RX_OFFLOAD_IPV4_CKSUM | 678 DEV_RX_OFFLOAD_UDP_CKSUM | 679 DEV_RX_OFFLOAD_TCP_CKSUM); 680 dev_info->tx_offload_capa = 681 (DEV_TX_OFFLOAD_IPV4_CKSUM | 682 DEV_TX_OFFLOAD_UDP_CKSUM | 683 DEV_TX_OFFLOAD_TCP_CKSUM); 684 } 685 686 static int 687 tap_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *tap_stats) 688 { 689 unsigned int i, imax; 690 unsigned long rx_total = 0, tx_total = 0, tx_err_total = 0; 691 unsigned long rx_bytes_total = 0, tx_bytes_total = 0; 692 unsigned long rx_nombuf = 0, ierrors = 0; 693 const struct pmd_internals *pmd = dev->data->dev_private; 694 695 /* rx queue statistics */ 696 imax = (dev->data->nb_rx_queues < RTE_ETHDEV_QUEUE_STAT_CNTRS) ? 697 dev->data->nb_rx_queues : RTE_ETHDEV_QUEUE_STAT_CNTRS; 698 for (i = 0; i < imax; i++) { 699 tap_stats->q_ipackets[i] = pmd->rxq[i].stats.ipackets; 700 tap_stats->q_ibytes[i] = pmd->rxq[i].stats.ibytes; 701 rx_total += tap_stats->q_ipackets[i]; 702 rx_bytes_total += tap_stats->q_ibytes[i]; 703 rx_nombuf += pmd->rxq[i].stats.rx_nombuf; 704 ierrors += pmd->rxq[i].stats.ierrors; 705 } 706 707 /* tx queue statistics */ 708 imax = (dev->data->nb_tx_queues < RTE_ETHDEV_QUEUE_STAT_CNTRS) ? 709 dev->data->nb_tx_queues : RTE_ETHDEV_QUEUE_STAT_CNTRS; 710 711 for (i = 0; i < imax; i++) { 712 tap_stats->q_opackets[i] = pmd->txq[i].stats.opackets; 713 tap_stats->q_errors[i] = pmd->txq[i].stats.errs; 714 tap_stats->q_obytes[i] = pmd->txq[i].stats.obytes; 715 tx_total += tap_stats->q_opackets[i]; 716 tx_err_total += tap_stats->q_errors[i]; 717 tx_bytes_total += tap_stats->q_obytes[i]; 718 } 719 720 tap_stats->ipackets = rx_total; 721 tap_stats->ibytes = rx_bytes_total; 722 tap_stats->ierrors = ierrors; 723 tap_stats->rx_nombuf = rx_nombuf; 724 tap_stats->opackets = tx_total; 725 tap_stats->oerrors = tx_err_total; 726 tap_stats->obytes = tx_bytes_total; 727 return 0; 728 } 729 730 static void 731 tap_stats_reset(struct rte_eth_dev *dev) 732 { 733 int i; 734 struct pmd_internals *pmd = dev->data->dev_private; 735 736 for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) { 737 pmd->rxq[i].stats.ipackets = 0; 738 pmd->rxq[i].stats.ibytes = 0; 739 pmd->rxq[i].stats.ierrors = 0; 740 pmd->rxq[i].stats.rx_nombuf = 0; 741 742 pmd->txq[i].stats.opackets = 0; 743 pmd->txq[i].stats.errs = 0; 744 pmd->txq[i].stats.obytes = 0; 745 } 746 } 747 748 static void 749 tap_dev_close(struct rte_eth_dev *dev) 750 { 751 int i; 752 struct pmd_internals *internals = dev->data->dev_private; 753 754 tap_link_set_down(dev); 755 tap_flow_flush(dev, NULL); 756 tap_flow_implicit_flush(internals, NULL); 757 758 for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) { 759 if (internals->rxq[i].fd != -1) { 760 close(internals->rxq[i].fd); 761 internals->rxq[i].fd = -1; 762 } 763 if (internals->txq[i].fd != -1) { 764 close(internals->txq[i].fd); 765 internals->txq[i].fd = -1; 766 } 767 } 768 769 if (internals->remote_if_index) { 770 /* Restore initial remote state */ 771 ioctl(internals->ioctl_sock, SIOCSIFFLAGS, 772 &internals->remote_initial_flags); 773 } 774 } 775 776 static void 777 tap_rx_queue_release(void *queue) 778 { 779 struct rx_queue *rxq = queue; 780 781 if (rxq && (rxq->fd > 0)) { 782 close(rxq->fd); 783 rxq->fd = -1; 784 rte_pktmbuf_free(rxq->pool); 785 rte_free(rxq->iovecs); 786 rxq->pool = NULL; 787 rxq->iovecs = NULL; 788 } 789 } 790 791 static void 792 tap_tx_queue_release(void *queue) 793 { 794 struct tx_queue *txq = queue; 795 796 if (txq && (txq->fd > 0)) { 797 close(txq->fd); 798 txq->fd = -1; 799 } 800 } 801 802 static int 803 tap_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused) 804 { 805 struct rte_eth_link *dev_link = &dev->data->dev_link; 806 struct pmd_internals *pmd = dev->data->dev_private; 807 struct ifreq ifr = { .ifr_flags = 0 }; 808 809 if (pmd->remote_if_index) { 810 tap_ioctl(pmd, SIOCGIFFLAGS, &ifr, 0, REMOTE_ONLY); 811 if (!(ifr.ifr_flags & IFF_UP) || 812 !(ifr.ifr_flags & IFF_RUNNING)) { 813 dev_link->link_status = ETH_LINK_DOWN; 814 return 0; 815 } 816 } 817 tap_ioctl(pmd, SIOCGIFFLAGS, &ifr, 0, LOCAL_ONLY); 818 dev_link->link_status = 819 ((ifr.ifr_flags & IFF_UP) && (ifr.ifr_flags & IFF_RUNNING) ? 820 ETH_LINK_UP : 821 ETH_LINK_DOWN); 822 return 0; 823 } 824 825 static void 826 tap_promisc_enable(struct rte_eth_dev *dev) 827 { 828 struct pmd_internals *pmd = dev->data->dev_private; 829 struct ifreq ifr = { .ifr_flags = IFF_PROMISC }; 830 831 dev->data->promiscuous = 1; 832 tap_ioctl(pmd, SIOCSIFFLAGS, &ifr, 1, LOCAL_AND_REMOTE); 833 if (pmd->remote_if_index && !pmd->flow_isolate) 834 tap_flow_implicit_create(pmd, TAP_REMOTE_PROMISC); 835 } 836 837 static void 838 tap_promisc_disable(struct rte_eth_dev *dev) 839 { 840 struct pmd_internals *pmd = dev->data->dev_private; 841 struct ifreq ifr = { .ifr_flags = IFF_PROMISC }; 842 843 dev->data->promiscuous = 0; 844 tap_ioctl(pmd, SIOCSIFFLAGS, &ifr, 0, LOCAL_AND_REMOTE); 845 if (pmd->remote_if_index && !pmd->flow_isolate) 846 tap_flow_implicit_destroy(pmd, TAP_REMOTE_PROMISC); 847 } 848 849 static void 850 tap_allmulti_enable(struct rte_eth_dev *dev) 851 { 852 struct pmd_internals *pmd = dev->data->dev_private; 853 struct ifreq ifr = { .ifr_flags = IFF_ALLMULTI }; 854 855 dev->data->all_multicast = 1; 856 tap_ioctl(pmd, SIOCSIFFLAGS, &ifr, 1, LOCAL_AND_REMOTE); 857 if (pmd->remote_if_index && !pmd->flow_isolate) 858 tap_flow_implicit_create(pmd, TAP_REMOTE_ALLMULTI); 859 } 860 861 static void 862 tap_allmulti_disable(struct rte_eth_dev *dev) 863 { 864 struct pmd_internals *pmd = dev->data->dev_private; 865 struct ifreq ifr = { .ifr_flags = IFF_ALLMULTI }; 866 867 dev->data->all_multicast = 0; 868 tap_ioctl(pmd, SIOCSIFFLAGS, &ifr, 0, LOCAL_AND_REMOTE); 869 if (pmd->remote_if_index && !pmd->flow_isolate) 870 tap_flow_implicit_destroy(pmd, TAP_REMOTE_ALLMULTI); 871 } 872 873 static void 874 tap_mac_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr) 875 { 876 struct pmd_internals *pmd = dev->data->dev_private; 877 enum ioctl_mode mode = LOCAL_ONLY; 878 struct ifreq ifr; 879 880 if (is_zero_ether_addr(mac_addr)) { 881 RTE_LOG(ERR, PMD, "%s: can't set an empty MAC address\n", 882 dev->device->name); 883 return; 884 } 885 /* Check the actual current MAC address on the tap netdevice */ 886 if (tap_ioctl(pmd, SIOCGIFHWADDR, &ifr, 0, LOCAL_ONLY) < 0) 887 return; 888 if (is_same_ether_addr((struct ether_addr *)&ifr.ifr_hwaddr.sa_data, 889 mac_addr)) 890 return; 891 /* Check the current MAC address on the remote */ 892 if (tap_ioctl(pmd, SIOCGIFHWADDR, &ifr, 0, REMOTE_ONLY) < 0) 893 return; 894 if (!is_same_ether_addr((struct ether_addr *)&ifr.ifr_hwaddr.sa_data, 895 mac_addr)) 896 mode = LOCAL_AND_REMOTE; 897 ifr.ifr_hwaddr.sa_family = AF_LOCAL; 898 rte_memcpy(ifr.ifr_hwaddr.sa_data, mac_addr, ETHER_ADDR_LEN); 899 if (tap_ioctl(pmd, SIOCSIFHWADDR, &ifr, 1, mode) < 0) 900 return; 901 rte_memcpy(&pmd->eth_addr, mac_addr, ETHER_ADDR_LEN); 902 if (pmd->remote_if_index && !pmd->flow_isolate) { 903 /* Replace MAC redirection rule after a MAC change */ 904 if (tap_flow_implicit_destroy(pmd, TAP_REMOTE_LOCAL_MAC) < 0) { 905 RTE_LOG(ERR, PMD, 906 "%s: Couldn't delete MAC redirection rule\n", 907 dev->device->name); 908 return; 909 } 910 if (tap_flow_implicit_create(pmd, TAP_REMOTE_LOCAL_MAC) < 0) 911 RTE_LOG(ERR, PMD, 912 "%s: Couldn't add MAC redirection rule\n", 913 dev->device->name); 914 } 915 } 916 917 static int 918 tap_setup_queue(struct rte_eth_dev *dev, 919 struct pmd_internals *internals, 920 uint16_t qid, 921 int is_rx) 922 { 923 int *fd; 924 int *other_fd; 925 const char *dir; 926 struct pmd_internals *pmd = dev->data->dev_private; 927 struct rx_queue *rx = &internals->rxq[qid]; 928 struct tx_queue *tx = &internals->txq[qid]; 929 930 if (is_rx) { 931 fd = &rx->fd; 932 other_fd = &tx->fd; 933 dir = "rx"; 934 } else { 935 fd = &tx->fd; 936 other_fd = &rx->fd; 937 dir = "tx"; 938 } 939 if (*fd != -1) { 940 /* fd for this queue already exists */ 941 RTE_LOG(DEBUG, PMD, "%s: fd %d for %s queue qid %d exists\n", 942 pmd->name, *fd, dir, qid); 943 } else if (*other_fd != -1) { 944 /* Only other_fd exists. dup it */ 945 *fd = dup(*other_fd); 946 if (*fd < 0) { 947 *fd = -1; 948 RTE_LOG(ERR, PMD, "%s: dup() failed.\n", 949 pmd->name); 950 return -1; 951 } 952 RTE_LOG(DEBUG, PMD, "%s: dup fd %d for %s queue qid %d (%d)\n", 953 pmd->name, *other_fd, dir, qid, *fd); 954 } else { 955 /* Both RX and TX fds do not exist (equal -1). Create fd */ 956 *fd = tun_alloc(pmd); 957 if (*fd < 0) { 958 *fd = -1; /* restore original value */ 959 RTE_LOG(ERR, PMD, "%s: tun_alloc() failed.\n", 960 pmd->name); 961 return -1; 962 } 963 RTE_LOG(DEBUG, PMD, "%s: add %s queue for qid %d fd %d\n", 964 pmd->name, dir, qid, *fd); 965 } 966 967 tx->mtu = &dev->data->mtu; 968 rx->rxmode = &dev->data->dev_conf.rxmode; 969 970 return *fd; 971 } 972 973 static int 974 tap_rx_queue_setup(struct rte_eth_dev *dev, 975 uint16_t rx_queue_id, 976 uint16_t nb_rx_desc, 977 unsigned int socket_id, 978 const struct rte_eth_rxconf *rx_conf __rte_unused, 979 struct rte_mempool *mp) 980 { 981 struct pmd_internals *internals = dev->data->dev_private; 982 struct rx_queue *rxq = &internals->rxq[rx_queue_id]; 983 struct rte_mbuf **tmp = &rxq->pool; 984 long iov_max = sysconf(_SC_IOV_MAX); 985 uint16_t nb_desc = RTE_MIN(nb_rx_desc, iov_max - 1); 986 struct iovec (*iovecs)[nb_desc + 1]; 987 int data_off = RTE_PKTMBUF_HEADROOM; 988 int ret = 0; 989 int fd; 990 int i; 991 992 if (rx_queue_id >= dev->data->nb_rx_queues || !mp) { 993 RTE_LOG(WARNING, PMD, 994 "nb_rx_queues %d too small or mempool NULL\n", 995 dev->data->nb_rx_queues); 996 return -1; 997 } 998 999 rxq->mp = mp; 1000 rxq->trigger_seen = 1; /* force initial burst */ 1001 rxq->in_port = dev->data->port_id; 1002 rxq->nb_rx_desc = nb_desc; 1003 iovecs = rte_zmalloc_socket(dev->device->name, sizeof(*iovecs), 0, 1004 socket_id); 1005 if (!iovecs) { 1006 RTE_LOG(WARNING, PMD, 1007 "%s: Couldn't allocate %d RX descriptors\n", 1008 dev->device->name, nb_desc); 1009 return -ENOMEM; 1010 } 1011 rxq->iovecs = iovecs; 1012 1013 dev->data->rx_queues[rx_queue_id] = rxq; 1014 fd = tap_setup_queue(dev, internals, rx_queue_id, 1); 1015 if (fd == -1) { 1016 ret = fd; 1017 goto error; 1018 } 1019 1020 (*rxq->iovecs)[0].iov_len = sizeof(struct tun_pi); 1021 (*rxq->iovecs)[0].iov_base = &rxq->pi; 1022 1023 for (i = 1; i <= nb_desc; i++) { 1024 *tmp = rte_pktmbuf_alloc(rxq->mp); 1025 if (!*tmp) { 1026 RTE_LOG(WARNING, PMD, 1027 "%s: couldn't allocate memory for queue %d\n", 1028 dev->device->name, rx_queue_id); 1029 ret = -ENOMEM; 1030 goto error; 1031 } 1032 (*rxq->iovecs)[i].iov_len = (*tmp)->buf_len - data_off; 1033 (*rxq->iovecs)[i].iov_base = 1034 (char *)(*tmp)->buf_addr + data_off; 1035 data_off = 0; 1036 tmp = &(*tmp)->next; 1037 } 1038 1039 RTE_LOG(DEBUG, PMD, " RX TAP device name %s, qid %d on fd %d\n", 1040 internals->name, rx_queue_id, internals->rxq[rx_queue_id].fd); 1041 1042 return 0; 1043 1044 error: 1045 rte_pktmbuf_free(rxq->pool); 1046 rxq->pool = NULL; 1047 rte_free(rxq->iovecs); 1048 rxq->iovecs = NULL; 1049 return ret; 1050 } 1051 1052 static int 1053 tap_tx_queue_setup(struct rte_eth_dev *dev, 1054 uint16_t tx_queue_id, 1055 uint16_t nb_tx_desc __rte_unused, 1056 unsigned int socket_id __rte_unused, 1057 const struct rte_eth_txconf *tx_conf __rte_unused) 1058 { 1059 struct pmd_internals *internals = dev->data->dev_private; 1060 int ret; 1061 1062 if (tx_queue_id >= dev->data->nb_tx_queues) 1063 return -1; 1064 1065 dev->data->tx_queues[tx_queue_id] = &internals->txq[tx_queue_id]; 1066 ret = tap_setup_queue(dev, internals, tx_queue_id, 0); 1067 if (ret == -1) 1068 return -1; 1069 1070 RTE_LOG(DEBUG, PMD, " TX TAP device name %s, qid %d on fd %d\n", 1071 internals->name, tx_queue_id, internals->txq[tx_queue_id].fd); 1072 1073 return 0; 1074 } 1075 1076 static int 1077 tap_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 1078 { 1079 struct pmd_internals *pmd = dev->data->dev_private; 1080 struct ifreq ifr = { .ifr_mtu = mtu }; 1081 int err = 0; 1082 1083 err = tap_ioctl(pmd, SIOCSIFMTU, &ifr, 1, LOCAL_AND_REMOTE); 1084 if (!err) 1085 dev->data->mtu = mtu; 1086 1087 return err; 1088 } 1089 1090 static int 1091 tap_set_mc_addr_list(struct rte_eth_dev *dev __rte_unused, 1092 struct ether_addr *mc_addr_set __rte_unused, 1093 uint32_t nb_mc_addr __rte_unused) 1094 { 1095 /* 1096 * Nothing to do actually: the tap has no filtering whatsoever, every 1097 * packet is received. 1098 */ 1099 return 0; 1100 } 1101 1102 static int 1103 tap_nl_msg_handler(struct nlmsghdr *nh, void *arg) 1104 { 1105 struct rte_eth_dev *dev = arg; 1106 struct pmd_internals *pmd = dev->data->dev_private; 1107 struct ifinfomsg *info = NLMSG_DATA(nh); 1108 1109 if (nh->nlmsg_type != RTM_NEWLINK || 1110 (info->ifi_index != pmd->if_index && 1111 info->ifi_index != pmd->remote_if_index)) 1112 return 0; 1113 return tap_link_update(dev, 0); 1114 } 1115 1116 static void 1117 tap_dev_intr_handler(void *cb_arg) 1118 { 1119 struct rte_eth_dev *dev = cb_arg; 1120 struct pmd_internals *pmd = dev->data->dev_private; 1121 1122 nl_recv(pmd->intr_handle.fd, tap_nl_msg_handler, dev); 1123 } 1124 1125 static int 1126 tap_intr_handle_set(struct rte_eth_dev *dev, int set) 1127 { 1128 struct pmd_internals *pmd = dev->data->dev_private; 1129 1130 /* In any case, disable interrupt if the conf is no longer there. */ 1131 if (!dev->data->dev_conf.intr_conf.lsc) { 1132 if (pmd->intr_handle.fd != -1) { 1133 nl_final(pmd->intr_handle.fd); 1134 rte_intr_callback_unregister(&pmd->intr_handle, 1135 tap_dev_intr_handler, dev); 1136 } 1137 return 0; 1138 } 1139 if (set) { 1140 pmd->intr_handle.fd = nl_init(RTMGRP_LINK); 1141 if (unlikely(pmd->intr_handle.fd == -1)) 1142 return -EBADF; 1143 return rte_intr_callback_register( 1144 &pmd->intr_handle, tap_dev_intr_handler, dev); 1145 } 1146 nl_final(pmd->intr_handle.fd); 1147 return rte_intr_callback_unregister(&pmd->intr_handle, 1148 tap_dev_intr_handler, dev); 1149 } 1150 1151 static const uint32_t* 1152 tap_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused) 1153 { 1154 static const uint32_t ptypes[] = { 1155 RTE_PTYPE_INNER_L2_ETHER, 1156 RTE_PTYPE_INNER_L2_ETHER_VLAN, 1157 RTE_PTYPE_INNER_L2_ETHER_QINQ, 1158 RTE_PTYPE_INNER_L3_IPV4, 1159 RTE_PTYPE_INNER_L3_IPV4_EXT, 1160 RTE_PTYPE_INNER_L3_IPV6, 1161 RTE_PTYPE_INNER_L3_IPV6_EXT, 1162 RTE_PTYPE_INNER_L4_FRAG, 1163 RTE_PTYPE_INNER_L4_UDP, 1164 RTE_PTYPE_INNER_L4_TCP, 1165 RTE_PTYPE_INNER_L4_SCTP, 1166 RTE_PTYPE_L2_ETHER, 1167 RTE_PTYPE_L2_ETHER_VLAN, 1168 RTE_PTYPE_L2_ETHER_QINQ, 1169 RTE_PTYPE_L3_IPV4, 1170 RTE_PTYPE_L3_IPV4_EXT, 1171 RTE_PTYPE_L3_IPV6_EXT, 1172 RTE_PTYPE_L3_IPV6, 1173 RTE_PTYPE_L4_FRAG, 1174 RTE_PTYPE_L4_UDP, 1175 RTE_PTYPE_L4_TCP, 1176 RTE_PTYPE_L4_SCTP, 1177 }; 1178 1179 return ptypes; 1180 } 1181 1182 static int 1183 tap_flow_ctrl_get(struct rte_eth_dev *dev __rte_unused, 1184 struct rte_eth_fc_conf *fc_conf) 1185 { 1186 fc_conf->mode = RTE_FC_NONE; 1187 return 0; 1188 } 1189 1190 static int 1191 tap_flow_ctrl_set(struct rte_eth_dev *dev __rte_unused, 1192 struct rte_eth_fc_conf *fc_conf) 1193 { 1194 if (fc_conf->mode != RTE_FC_NONE) 1195 return -ENOTSUP; 1196 return 0; 1197 } 1198 1199 static const struct eth_dev_ops ops = { 1200 .dev_start = tap_dev_start, 1201 .dev_stop = tap_dev_stop, 1202 .dev_close = tap_dev_close, 1203 .dev_configure = tap_dev_configure, 1204 .dev_infos_get = tap_dev_info, 1205 .rx_queue_setup = tap_rx_queue_setup, 1206 .tx_queue_setup = tap_tx_queue_setup, 1207 .rx_queue_release = tap_rx_queue_release, 1208 .tx_queue_release = tap_tx_queue_release, 1209 .flow_ctrl_get = tap_flow_ctrl_get, 1210 .flow_ctrl_set = tap_flow_ctrl_set, 1211 .link_update = tap_link_update, 1212 .dev_set_link_up = tap_link_set_up, 1213 .dev_set_link_down = tap_link_set_down, 1214 .promiscuous_enable = tap_promisc_enable, 1215 .promiscuous_disable = tap_promisc_disable, 1216 .allmulticast_enable = tap_allmulti_enable, 1217 .allmulticast_disable = tap_allmulti_disable, 1218 .mac_addr_set = tap_mac_set, 1219 .mtu_set = tap_mtu_set, 1220 .set_mc_addr_list = tap_set_mc_addr_list, 1221 .stats_get = tap_stats_get, 1222 .stats_reset = tap_stats_reset, 1223 .dev_supported_ptypes_get = tap_dev_supported_ptypes_get, 1224 .filter_ctrl = tap_dev_filter_ctrl, 1225 }; 1226 1227 static int 1228 eth_dev_tap_create(struct rte_vdev_device *vdev, char *tap_name, 1229 char *remote_iface, int fixed_mac_type) 1230 { 1231 int numa_node = rte_socket_id(); 1232 struct rte_eth_dev *dev; 1233 struct pmd_internals *pmd; 1234 struct rte_eth_dev_data *data; 1235 struct ifreq ifr; 1236 int i; 1237 1238 RTE_LOG(DEBUG, PMD, " TAP device on numa %u\n", rte_socket_id()); 1239 1240 data = rte_zmalloc_socket(tap_name, sizeof(*data), 0, numa_node); 1241 if (!data) { 1242 RTE_LOG(ERR, PMD, "TAP Failed to allocate data\n"); 1243 goto error_exit_nodev; 1244 } 1245 1246 dev = rte_eth_vdev_allocate(vdev, sizeof(*pmd)); 1247 if (!dev) { 1248 RTE_LOG(ERR, PMD, "TAP Unable to allocate device struct\n"); 1249 goto error_exit_nodev; 1250 } 1251 1252 pmd = dev->data->dev_private; 1253 pmd->dev = dev; 1254 snprintf(pmd->name, sizeof(pmd->name), "%s", tap_name); 1255 1256 pmd->ioctl_sock = socket(AF_INET, SOCK_DGRAM, 0); 1257 if (pmd->ioctl_sock == -1) { 1258 RTE_LOG(ERR, PMD, 1259 "TAP Unable to get a socket for management: %s\n", 1260 strerror(errno)); 1261 goto error_exit; 1262 } 1263 1264 /* Setup some default values */ 1265 rte_memcpy(data, dev->data, sizeof(*data)); 1266 data->dev_private = pmd; 1267 data->dev_flags = RTE_ETH_DEV_INTR_LSC; 1268 data->numa_node = numa_node; 1269 1270 data->dev_link = pmd_link; 1271 data->mac_addrs = &pmd->eth_addr; 1272 /* Set the number of RX and TX queues */ 1273 data->nb_rx_queues = 0; 1274 data->nb_tx_queues = 0; 1275 1276 dev->data = data; 1277 dev->dev_ops = &ops; 1278 dev->rx_pkt_burst = pmd_rx_burst; 1279 dev->tx_pkt_burst = pmd_tx_burst; 1280 1281 pmd->intr_handle.type = RTE_INTR_HANDLE_EXT; 1282 pmd->intr_handle.fd = -1; 1283 1284 /* Presetup the fds to -1 as being not valid */ 1285 for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) { 1286 pmd->rxq[i].fd = -1; 1287 pmd->txq[i].fd = -1; 1288 } 1289 1290 if (fixed_mac_type) { 1291 /* fixed mac = 00:64:74:61:70:<iface_idx> */ 1292 static int iface_idx; 1293 char mac[ETHER_ADDR_LEN] = "\0dtap"; 1294 1295 mac[ETHER_ADDR_LEN - 1] = iface_idx++; 1296 rte_memcpy(&pmd->eth_addr, mac, ETHER_ADDR_LEN); 1297 } else { 1298 eth_random_addr((uint8_t *)&pmd->eth_addr); 1299 } 1300 1301 /* Immediately create the netdevice (this will create the 1st queue). */ 1302 /* rx queue */ 1303 if (tap_setup_queue(dev, pmd, 0, 1) == -1) 1304 goto error_exit; 1305 /* tx queue */ 1306 if (tap_setup_queue(dev, pmd, 0, 0) == -1) 1307 goto error_exit; 1308 1309 ifr.ifr_mtu = dev->data->mtu; 1310 if (tap_ioctl(pmd, SIOCSIFMTU, &ifr, 1, LOCAL_AND_REMOTE) < 0) 1311 goto error_exit; 1312 1313 memset(&ifr, 0, sizeof(struct ifreq)); 1314 ifr.ifr_hwaddr.sa_family = AF_LOCAL; 1315 rte_memcpy(ifr.ifr_hwaddr.sa_data, &pmd->eth_addr, ETHER_ADDR_LEN); 1316 if (tap_ioctl(pmd, SIOCSIFHWADDR, &ifr, 0, LOCAL_ONLY) < 0) 1317 goto error_exit; 1318 1319 /* 1320 * Set up everything related to rte_flow: 1321 * - netlink socket 1322 * - tap / remote if_index 1323 * - mandatory QDISCs 1324 * - rte_flow actual/implicit lists 1325 * - implicit rules 1326 */ 1327 pmd->nlsk_fd = nl_init(0); 1328 if (pmd->nlsk_fd == -1) { 1329 RTE_LOG(WARNING, PMD, "%s: failed to create netlink socket.\n", 1330 pmd->name); 1331 goto disable_rte_flow; 1332 } 1333 pmd->if_index = if_nametoindex(pmd->name); 1334 if (!pmd->if_index) { 1335 RTE_LOG(ERR, PMD, "%s: failed to get if_index.\n", pmd->name); 1336 goto disable_rte_flow; 1337 } 1338 if (qdisc_create_multiq(pmd->nlsk_fd, pmd->if_index) < 0) { 1339 RTE_LOG(ERR, PMD, "%s: failed to create multiq qdisc.\n", 1340 pmd->name); 1341 goto disable_rte_flow; 1342 } 1343 if (qdisc_create_ingress(pmd->nlsk_fd, pmd->if_index) < 0) { 1344 RTE_LOG(ERR, PMD, "%s: failed to create ingress qdisc.\n", 1345 pmd->name); 1346 goto disable_rte_flow; 1347 } 1348 LIST_INIT(&pmd->flows); 1349 1350 if (strlen(remote_iface)) { 1351 pmd->remote_if_index = if_nametoindex(remote_iface); 1352 if (!pmd->remote_if_index) { 1353 RTE_LOG(ERR, PMD, "%s: failed to get %s if_index.\n", 1354 pmd->name, remote_iface); 1355 goto error_remote; 1356 } 1357 snprintf(pmd->remote_iface, RTE_ETH_NAME_MAX_LEN, 1358 "%s", remote_iface); 1359 1360 /* Save state of remote device */ 1361 tap_ioctl(pmd, SIOCGIFFLAGS, &pmd->remote_initial_flags, 0, REMOTE_ONLY); 1362 1363 /* Replicate remote MAC address */ 1364 if (tap_ioctl(pmd, SIOCGIFHWADDR, &ifr, 0, REMOTE_ONLY) < 0) { 1365 RTE_LOG(ERR, PMD, "%s: failed to get %s MAC address.\n", 1366 pmd->name, pmd->remote_iface); 1367 goto error_remote; 1368 } 1369 rte_memcpy(&pmd->eth_addr, ifr.ifr_hwaddr.sa_data, 1370 ETHER_ADDR_LEN); 1371 /* The desired MAC is already in ifreq after SIOCGIFHWADDR. */ 1372 if (tap_ioctl(pmd, SIOCSIFHWADDR, &ifr, 0, LOCAL_ONLY) < 0) { 1373 RTE_LOG(ERR, PMD, "%s: failed to get %s MAC address.\n", 1374 pmd->name, remote_iface); 1375 goto error_remote; 1376 } 1377 1378 /* 1379 * Flush usually returns negative value because it tries to 1380 * delete every QDISC (and on a running device, one QDISC at 1381 * least is needed). Ignore negative return value. 1382 */ 1383 qdisc_flush(pmd->nlsk_fd, pmd->remote_if_index); 1384 if (qdisc_create_ingress(pmd->nlsk_fd, 1385 pmd->remote_if_index) < 0) { 1386 RTE_LOG(ERR, PMD, "%s: failed to create ingress qdisc.\n", 1387 pmd->remote_iface); 1388 goto error_remote; 1389 } 1390 LIST_INIT(&pmd->implicit_flows); 1391 if (tap_flow_implicit_create(pmd, TAP_REMOTE_TX) < 0 || 1392 tap_flow_implicit_create(pmd, TAP_REMOTE_LOCAL_MAC) < 0 || 1393 tap_flow_implicit_create(pmd, TAP_REMOTE_BROADCAST) < 0 || 1394 tap_flow_implicit_create(pmd, TAP_REMOTE_BROADCASTV6) < 0) { 1395 RTE_LOG(ERR, PMD, 1396 "%s: failed to create implicit rules.\n", 1397 pmd->name); 1398 goto error_remote; 1399 } 1400 } 1401 1402 return 0; 1403 1404 disable_rte_flow: 1405 RTE_LOG(ERR, PMD, " Disabling rte flow support: %s(%d)\n", 1406 strerror(errno), errno); 1407 if (strlen(remote_iface)) { 1408 RTE_LOG(ERR, PMD, "Remote feature requires flow support.\n"); 1409 goto error_exit; 1410 } 1411 return 0; 1412 1413 error_remote: 1414 RTE_LOG(ERR, PMD, " Can't set up remote feature: %s(%d)\n", 1415 strerror(errno), errno); 1416 tap_flow_implicit_flush(pmd, NULL); 1417 1418 error_exit: 1419 if (pmd->ioctl_sock > 0) 1420 close(pmd->ioctl_sock); 1421 rte_eth_dev_release_port(dev); 1422 1423 error_exit_nodev: 1424 RTE_LOG(ERR, PMD, "TAP Unable to initialize %s\n", 1425 rte_vdev_device_name(vdev)); 1426 1427 rte_free(data); 1428 return -EINVAL; 1429 } 1430 1431 static int 1432 set_interface_name(const char *key __rte_unused, 1433 const char *value, 1434 void *extra_args) 1435 { 1436 char *name = (char *)extra_args; 1437 1438 if (value) 1439 snprintf(name, RTE_ETH_NAME_MAX_LEN - 1, "%s", value); 1440 else 1441 snprintf(name, RTE_ETH_NAME_MAX_LEN - 1, "%s%d", 1442 DEFAULT_TAP_NAME, (tap_unit - 1)); 1443 1444 return 0; 1445 } 1446 1447 static int 1448 set_interface_speed(const char *key __rte_unused, 1449 const char *value, 1450 void *extra_args) 1451 { 1452 *(int *)extra_args = (value) ? atoi(value) : ETH_SPEED_NUM_10G; 1453 1454 return 0; 1455 } 1456 1457 static int 1458 set_remote_iface(const char *key __rte_unused, 1459 const char *value, 1460 void *extra_args) 1461 { 1462 char *name = (char *)extra_args; 1463 1464 if (value) 1465 snprintf(name, RTE_ETH_NAME_MAX_LEN, "%s", value); 1466 1467 return 0; 1468 } 1469 1470 static int 1471 set_mac_type(const char *key __rte_unused, 1472 const char *value, 1473 void *extra_args) 1474 { 1475 if (value && 1476 !strncasecmp(ETH_TAP_MAC_FIXED, value, strlen(ETH_TAP_MAC_FIXED))) 1477 *(int *)extra_args = 1; 1478 return 0; 1479 } 1480 1481 /* Open a TAP interface device. 1482 */ 1483 static int 1484 rte_pmd_tap_probe(struct rte_vdev_device *dev) 1485 { 1486 const char *name, *params; 1487 int ret; 1488 struct rte_kvargs *kvlist = NULL; 1489 int speed; 1490 char tap_name[RTE_ETH_NAME_MAX_LEN]; 1491 char remote_iface[RTE_ETH_NAME_MAX_LEN]; 1492 int fixed_mac_type = 0; 1493 1494 name = rte_vdev_device_name(dev); 1495 params = rte_vdev_device_args(dev); 1496 1497 speed = ETH_SPEED_NUM_10G; 1498 snprintf(tap_name, sizeof(tap_name), "%s%d", 1499 DEFAULT_TAP_NAME, tap_unit++); 1500 memset(remote_iface, 0, RTE_ETH_NAME_MAX_LEN); 1501 1502 if (params && (params[0] != '\0')) { 1503 RTE_LOG(DEBUG, PMD, "parameters (%s)\n", params); 1504 1505 kvlist = rte_kvargs_parse(params, valid_arguments); 1506 if (kvlist) { 1507 if (rte_kvargs_count(kvlist, ETH_TAP_SPEED_ARG) == 1) { 1508 ret = rte_kvargs_process(kvlist, 1509 ETH_TAP_SPEED_ARG, 1510 &set_interface_speed, 1511 &speed); 1512 if (ret == -1) 1513 goto leave; 1514 } 1515 1516 if (rte_kvargs_count(kvlist, ETH_TAP_IFACE_ARG) == 1) { 1517 ret = rte_kvargs_process(kvlist, 1518 ETH_TAP_IFACE_ARG, 1519 &set_interface_name, 1520 tap_name); 1521 if (ret == -1) 1522 goto leave; 1523 } 1524 1525 if (rte_kvargs_count(kvlist, ETH_TAP_REMOTE_ARG) == 1) { 1526 ret = rte_kvargs_process(kvlist, 1527 ETH_TAP_REMOTE_ARG, 1528 &set_remote_iface, 1529 remote_iface); 1530 if (ret == -1) 1531 goto leave; 1532 } 1533 1534 if (rte_kvargs_count(kvlist, ETH_TAP_MAC_ARG) == 1) { 1535 ret = rte_kvargs_process(kvlist, 1536 ETH_TAP_MAC_ARG, 1537 &set_mac_type, 1538 &fixed_mac_type); 1539 if (ret == -1) 1540 goto leave; 1541 } 1542 } 1543 } 1544 pmd_link.link_speed = speed; 1545 1546 RTE_LOG(NOTICE, PMD, "Initializing pmd_tap for %s as %s\n", 1547 name, tap_name); 1548 1549 ret = eth_dev_tap_create(dev, tap_name, remote_iface, fixed_mac_type); 1550 1551 leave: 1552 if (ret == -1) { 1553 RTE_LOG(ERR, PMD, "Failed to create pmd for %s as %s\n", 1554 name, tap_name); 1555 tap_unit--; /* Restore the unit number */ 1556 } 1557 rte_kvargs_free(kvlist); 1558 1559 return ret; 1560 } 1561 1562 /* detach a TAP device. 1563 */ 1564 static int 1565 rte_pmd_tap_remove(struct rte_vdev_device *dev) 1566 { 1567 struct rte_eth_dev *eth_dev = NULL; 1568 struct pmd_internals *internals; 1569 int i; 1570 1571 RTE_LOG(DEBUG, PMD, "Closing TUN/TAP Ethernet device on numa %u\n", 1572 rte_socket_id()); 1573 1574 /* find the ethdev entry */ 1575 eth_dev = rte_eth_dev_allocated(rte_vdev_device_name(dev)); 1576 if (!eth_dev) 1577 return 0; 1578 1579 internals = eth_dev->data->dev_private; 1580 if (internals->nlsk_fd) { 1581 tap_flow_flush(eth_dev, NULL); 1582 tap_flow_implicit_flush(internals, NULL); 1583 nl_final(internals->nlsk_fd); 1584 } 1585 for (i = 0; i < RTE_PMD_TAP_MAX_QUEUES; i++) { 1586 if (internals->rxq[i].fd != -1) { 1587 close(internals->rxq[i].fd); 1588 internals->rxq[i].fd = -1; 1589 } 1590 if (internals->txq[i].fd != -1) { 1591 close(internals->txq[i].fd); 1592 internals->txq[i].fd = -1; 1593 } 1594 } 1595 1596 close(internals->ioctl_sock); 1597 rte_free(eth_dev->data->dev_private); 1598 rte_free(eth_dev->data); 1599 1600 rte_eth_dev_release_port(eth_dev); 1601 1602 return 0; 1603 } 1604 1605 static struct rte_vdev_driver pmd_tap_drv = { 1606 .probe = rte_pmd_tap_probe, 1607 .remove = rte_pmd_tap_remove, 1608 }; 1609 RTE_PMD_REGISTER_VDEV(net_tap, pmd_tap_drv); 1610 RTE_PMD_REGISTER_ALIAS(net_tap, eth_tap); 1611 RTE_PMD_REGISTER_PARAM_STRING(net_tap, 1612 ETH_TAP_IFACE_ARG "=<string> " 1613 ETH_TAP_SPEED_ARG "=<int> " 1614 ETH_TAP_MAC_ARG "=" ETH_TAP_MAC_FIXED " " 1615 ETH_TAP_REMOTE_ARG "=<string>"); 1616