1 /* 2 * Copyright (C) 2017 Netronome Systems, Inc. 3 * 4 * This software is licensed under the GNU General License Version 2, 5 * June 1991 as shown in the file COPYING in the top-level directory of this 6 * source tree. 7 * 8 * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" 9 * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, 10 * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 11 * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE 12 * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME 13 * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 14 */ 15 16 #include <linux/debugfs.h> 17 #include <linux/etherdevice.h> 18 #include <linux/kernel.h> 19 #include <linux/module.h> 20 #include <linux/netdevice.h> 21 #include <linux/slab.h> 22 #include <net/netdev_queues.h> 23 #include <net/page_pool/helpers.h> 24 #include <net/netlink.h> 25 #include <net/net_shaper.h> 26 #include <net/pkt_cls.h> 27 #include <net/rtnetlink.h> 28 #include <net/udp_tunnel.h> 29 30 #include "netdevsim.h" 31 32 #define NSIM_RING_SIZE 256 33 34 static int nsim_napi_rx(struct nsim_rq *rq, struct sk_buff *skb) 35 { 36 if (skb_queue_len(&rq->skb_queue) > NSIM_RING_SIZE) { 37 dev_kfree_skb_any(skb); 38 return NET_RX_DROP; 39 } 40 41 skb_queue_tail(&rq->skb_queue, skb); 42 return NET_RX_SUCCESS; 43 } 44 45 static int nsim_forward_skb(struct net_device *dev, struct sk_buff *skb, 46 struct nsim_rq *rq) 47 { 48 return __dev_forward_skb(dev, skb) ?: nsim_napi_rx(rq, skb); 49 } 50 51 static netdev_tx_t nsim_start_xmit(struct sk_buff *skb, struct net_device *dev) 52 { 53 struct netdevsim *ns = netdev_priv(dev); 54 struct net_device *peer_dev; 55 unsigned int len = skb->len; 56 struct netdevsim *peer_ns; 57 struct nsim_rq *rq; 58 int rxq; 59 60 rcu_read_lock(); 61 if (!nsim_ipsec_tx(ns, skb)) 62 goto out_drop_free; 63 64 peer_ns = rcu_dereference(ns->peer); 65 if (!peer_ns) 66 goto out_drop_free; 67 68 peer_dev = peer_ns->netdev; 69 rxq = skb_get_queue_mapping(skb); 70 if (rxq >= peer_dev->num_rx_queues) 71 rxq = rxq % peer_dev->num_rx_queues; 72 rq = peer_ns->rq[rxq]; 73 74 skb_tx_timestamp(skb); 75 if (unlikely(nsim_forward_skb(peer_dev, skb, rq) == NET_RX_DROP)) 76 goto out_drop_cnt; 77 78 napi_schedule(&rq->napi); 79 80 rcu_read_unlock(); 81 u64_stats_update_begin(&ns->syncp); 82 ns->tx_packets++; 83 ns->tx_bytes += len; 84 u64_stats_update_end(&ns->syncp); 85 return NETDEV_TX_OK; 86 87 out_drop_free: 88 dev_kfree_skb(skb); 89 out_drop_cnt: 90 rcu_read_unlock(); 91 u64_stats_update_begin(&ns->syncp); 92 ns->tx_dropped++; 93 u64_stats_update_end(&ns->syncp); 94 return NETDEV_TX_OK; 95 } 96 97 static void nsim_set_rx_mode(struct net_device *dev) 98 { 99 } 100 101 static int nsim_change_mtu(struct net_device *dev, int new_mtu) 102 { 103 struct netdevsim *ns = netdev_priv(dev); 104 105 if (ns->xdp.prog && new_mtu > NSIM_XDP_MAX_MTU) 106 return -EBUSY; 107 108 WRITE_ONCE(dev->mtu, new_mtu); 109 110 return 0; 111 } 112 113 static void 114 nsim_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 115 { 116 struct netdevsim *ns = netdev_priv(dev); 117 unsigned int start; 118 119 do { 120 start = u64_stats_fetch_begin(&ns->syncp); 121 stats->tx_bytes = ns->tx_bytes; 122 stats->tx_packets = ns->tx_packets; 123 stats->tx_dropped = ns->tx_dropped; 124 } while (u64_stats_fetch_retry(&ns->syncp, start)); 125 } 126 127 static int 128 nsim_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv) 129 { 130 return nsim_bpf_setup_tc_block_cb(type, type_data, cb_priv); 131 } 132 133 static int nsim_set_vf_mac(struct net_device *dev, int vf, u8 *mac) 134 { 135 struct netdevsim *ns = netdev_priv(dev); 136 struct nsim_dev *nsim_dev = ns->nsim_dev; 137 138 /* Only refuse multicast addresses, zero address can mean unset/any. */ 139 if (vf >= nsim_dev_get_vfs(nsim_dev) || is_multicast_ether_addr(mac)) 140 return -EINVAL; 141 memcpy(nsim_dev->vfconfigs[vf].vf_mac, mac, ETH_ALEN); 142 143 return 0; 144 } 145 146 static int nsim_set_vf_vlan(struct net_device *dev, int vf, 147 u16 vlan, u8 qos, __be16 vlan_proto) 148 { 149 struct netdevsim *ns = netdev_priv(dev); 150 struct nsim_dev *nsim_dev = ns->nsim_dev; 151 152 if (vf >= nsim_dev_get_vfs(nsim_dev) || vlan > 4095 || qos > 7) 153 return -EINVAL; 154 155 nsim_dev->vfconfigs[vf].vlan = vlan; 156 nsim_dev->vfconfigs[vf].qos = qos; 157 nsim_dev->vfconfigs[vf].vlan_proto = vlan_proto; 158 159 return 0; 160 } 161 162 static int nsim_set_vf_rate(struct net_device *dev, int vf, int min, int max) 163 { 164 struct netdevsim *ns = netdev_priv(dev); 165 struct nsim_dev *nsim_dev = ns->nsim_dev; 166 167 if (nsim_esw_mode_is_switchdev(ns->nsim_dev)) { 168 pr_err("Not supported in switchdev mode. Please use devlink API.\n"); 169 return -EOPNOTSUPP; 170 } 171 172 if (vf >= nsim_dev_get_vfs(nsim_dev)) 173 return -EINVAL; 174 175 nsim_dev->vfconfigs[vf].min_tx_rate = min; 176 nsim_dev->vfconfigs[vf].max_tx_rate = max; 177 178 return 0; 179 } 180 181 static int nsim_set_vf_spoofchk(struct net_device *dev, int vf, bool val) 182 { 183 struct netdevsim *ns = netdev_priv(dev); 184 struct nsim_dev *nsim_dev = ns->nsim_dev; 185 186 if (vf >= nsim_dev_get_vfs(nsim_dev)) 187 return -EINVAL; 188 nsim_dev->vfconfigs[vf].spoofchk_enabled = val; 189 190 return 0; 191 } 192 193 static int nsim_set_vf_rss_query_en(struct net_device *dev, int vf, bool val) 194 { 195 struct netdevsim *ns = netdev_priv(dev); 196 struct nsim_dev *nsim_dev = ns->nsim_dev; 197 198 if (vf >= nsim_dev_get_vfs(nsim_dev)) 199 return -EINVAL; 200 nsim_dev->vfconfigs[vf].rss_query_enabled = val; 201 202 return 0; 203 } 204 205 static int nsim_set_vf_trust(struct net_device *dev, int vf, bool val) 206 { 207 struct netdevsim *ns = netdev_priv(dev); 208 struct nsim_dev *nsim_dev = ns->nsim_dev; 209 210 if (vf >= nsim_dev_get_vfs(nsim_dev)) 211 return -EINVAL; 212 nsim_dev->vfconfigs[vf].trusted = val; 213 214 return 0; 215 } 216 217 static int 218 nsim_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivi) 219 { 220 struct netdevsim *ns = netdev_priv(dev); 221 struct nsim_dev *nsim_dev = ns->nsim_dev; 222 223 if (vf >= nsim_dev_get_vfs(nsim_dev)) 224 return -EINVAL; 225 226 ivi->vf = vf; 227 ivi->linkstate = nsim_dev->vfconfigs[vf].link_state; 228 ivi->min_tx_rate = nsim_dev->vfconfigs[vf].min_tx_rate; 229 ivi->max_tx_rate = nsim_dev->vfconfigs[vf].max_tx_rate; 230 ivi->vlan = nsim_dev->vfconfigs[vf].vlan; 231 ivi->vlan_proto = nsim_dev->vfconfigs[vf].vlan_proto; 232 ivi->qos = nsim_dev->vfconfigs[vf].qos; 233 memcpy(&ivi->mac, nsim_dev->vfconfigs[vf].vf_mac, ETH_ALEN); 234 ivi->spoofchk = nsim_dev->vfconfigs[vf].spoofchk_enabled; 235 ivi->trusted = nsim_dev->vfconfigs[vf].trusted; 236 ivi->rss_query_en = nsim_dev->vfconfigs[vf].rss_query_enabled; 237 238 return 0; 239 } 240 241 static int nsim_set_vf_link_state(struct net_device *dev, int vf, int state) 242 { 243 struct netdevsim *ns = netdev_priv(dev); 244 struct nsim_dev *nsim_dev = ns->nsim_dev; 245 246 if (vf >= nsim_dev_get_vfs(nsim_dev)) 247 return -EINVAL; 248 249 switch (state) { 250 case IFLA_VF_LINK_STATE_AUTO: 251 case IFLA_VF_LINK_STATE_ENABLE: 252 case IFLA_VF_LINK_STATE_DISABLE: 253 break; 254 default: 255 return -EINVAL; 256 } 257 258 nsim_dev->vfconfigs[vf].link_state = state; 259 260 return 0; 261 } 262 263 static void nsim_taprio_stats(struct tc_taprio_qopt_stats *stats) 264 { 265 stats->window_drops = 0; 266 stats->tx_overruns = 0; 267 } 268 269 static int nsim_setup_tc_taprio(struct net_device *dev, 270 struct tc_taprio_qopt_offload *offload) 271 { 272 int err = 0; 273 274 switch (offload->cmd) { 275 case TAPRIO_CMD_REPLACE: 276 case TAPRIO_CMD_DESTROY: 277 break; 278 case TAPRIO_CMD_STATS: 279 nsim_taprio_stats(&offload->stats); 280 break; 281 default: 282 err = -EOPNOTSUPP; 283 } 284 285 return err; 286 } 287 288 static LIST_HEAD(nsim_block_cb_list); 289 290 static int 291 nsim_setup_tc(struct net_device *dev, enum tc_setup_type type, void *type_data) 292 { 293 struct netdevsim *ns = netdev_priv(dev); 294 295 switch (type) { 296 case TC_SETUP_QDISC_TAPRIO: 297 return nsim_setup_tc_taprio(dev, type_data); 298 case TC_SETUP_BLOCK: 299 return flow_block_cb_setup_simple(type_data, 300 &nsim_block_cb_list, 301 nsim_setup_tc_block_cb, 302 ns, ns, true); 303 default: 304 return -EOPNOTSUPP; 305 } 306 } 307 308 static int 309 nsim_set_features(struct net_device *dev, netdev_features_t features) 310 { 311 struct netdevsim *ns = netdev_priv(dev); 312 313 if ((dev->features & NETIF_F_HW_TC) > (features & NETIF_F_HW_TC)) 314 return nsim_bpf_disable_tc(ns); 315 316 return 0; 317 } 318 319 static int nsim_get_iflink(const struct net_device *dev) 320 { 321 struct netdevsim *nsim, *peer; 322 int iflink; 323 324 nsim = netdev_priv(dev); 325 326 rcu_read_lock(); 327 peer = rcu_dereference(nsim->peer); 328 iflink = peer ? READ_ONCE(peer->netdev->ifindex) : 329 READ_ONCE(dev->ifindex); 330 rcu_read_unlock(); 331 332 return iflink; 333 } 334 335 static int nsim_rcv(struct nsim_rq *rq, int budget) 336 { 337 struct sk_buff *skb; 338 int i; 339 340 for (i = 0; i < budget; i++) { 341 if (skb_queue_empty(&rq->skb_queue)) 342 break; 343 344 skb = skb_dequeue(&rq->skb_queue); 345 netif_receive_skb(skb); 346 } 347 348 return i; 349 } 350 351 static int nsim_poll(struct napi_struct *napi, int budget) 352 { 353 struct nsim_rq *rq = container_of(napi, struct nsim_rq, napi); 354 int done; 355 356 done = nsim_rcv(rq, budget); 357 napi_complete(napi); 358 359 return done; 360 } 361 362 static int nsim_create_page_pool(struct page_pool **p, struct napi_struct *napi) 363 { 364 struct page_pool_params params = { 365 .order = 0, 366 .pool_size = NSIM_RING_SIZE, 367 .nid = NUMA_NO_NODE, 368 .dev = &napi->dev->dev, 369 .napi = napi, 370 .dma_dir = DMA_BIDIRECTIONAL, 371 .netdev = napi->dev, 372 }; 373 struct page_pool *pool; 374 375 pool = page_pool_create(¶ms); 376 if (IS_ERR(pool)) 377 return PTR_ERR(pool); 378 379 *p = pool; 380 return 0; 381 } 382 383 static int nsim_init_napi(struct netdevsim *ns) 384 { 385 struct net_device *dev = ns->netdev; 386 struct nsim_rq *rq; 387 int err, i; 388 389 for (i = 0; i < dev->num_rx_queues; i++) { 390 rq = ns->rq[i]; 391 392 netif_napi_add_config(dev, &rq->napi, nsim_poll, i); 393 } 394 395 for (i = 0; i < dev->num_rx_queues; i++) { 396 rq = ns->rq[i]; 397 398 err = nsim_create_page_pool(&rq->page_pool, &rq->napi); 399 if (err) 400 goto err_pp_destroy; 401 } 402 403 return 0; 404 405 err_pp_destroy: 406 while (i--) { 407 page_pool_destroy(ns->rq[i]->page_pool); 408 ns->rq[i]->page_pool = NULL; 409 } 410 411 for (i = 0; i < dev->num_rx_queues; i++) 412 __netif_napi_del(&ns->rq[i]->napi); 413 414 return err; 415 } 416 417 static void nsim_enable_napi(struct netdevsim *ns) 418 { 419 struct net_device *dev = ns->netdev; 420 int i; 421 422 for (i = 0; i < dev->num_rx_queues; i++) { 423 struct nsim_rq *rq = ns->rq[i]; 424 425 netif_queue_set_napi(dev, i, NETDEV_QUEUE_TYPE_RX, &rq->napi); 426 napi_enable(&rq->napi); 427 } 428 } 429 430 static int nsim_open(struct net_device *dev) 431 { 432 struct netdevsim *ns = netdev_priv(dev); 433 int err; 434 435 err = nsim_init_napi(ns); 436 if (err) 437 return err; 438 439 nsim_enable_napi(ns); 440 441 return 0; 442 } 443 444 static void nsim_del_napi(struct netdevsim *ns) 445 { 446 struct net_device *dev = ns->netdev; 447 int i; 448 449 for (i = 0; i < dev->num_rx_queues; i++) { 450 struct nsim_rq *rq = ns->rq[i]; 451 452 napi_disable(&rq->napi); 453 __netif_napi_del(&rq->napi); 454 } 455 synchronize_net(); 456 457 for (i = 0; i < dev->num_rx_queues; i++) { 458 page_pool_destroy(ns->rq[i]->page_pool); 459 ns->rq[i]->page_pool = NULL; 460 } 461 } 462 463 static int nsim_stop(struct net_device *dev) 464 { 465 struct netdevsim *ns = netdev_priv(dev); 466 struct netdevsim *peer; 467 468 netif_carrier_off(dev); 469 peer = rtnl_dereference(ns->peer); 470 if (peer) 471 netif_carrier_off(peer->netdev); 472 473 nsim_del_napi(ns); 474 475 return 0; 476 } 477 478 static int nsim_shaper_set(struct net_shaper_binding *binding, 479 const struct net_shaper *shaper, 480 struct netlink_ext_ack *extack) 481 { 482 return 0; 483 } 484 485 static int nsim_shaper_del(struct net_shaper_binding *binding, 486 const struct net_shaper_handle *handle, 487 struct netlink_ext_ack *extack) 488 { 489 return 0; 490 } 491 492 static int nsim_shaper_group(struct net_shaper_binding *binding, 493 int leaves_count, 494 const struct net_shaper *leaves, 495 const struct net_shaper *root, 496 struct netlink_ext_ack *extack) 497 { 498 return 0; 499 } 500 501 static void nsim_shaper_cap(struct net_shaper_binding *binding, 502 enum net_shaper_scope scope, 503 unsigned long *flags) 504 { 505 *flags = ULONG_MAX; 506 } 507 508 static const struct net_shaper_ops nsim_shaper_ops = { 509 .set = nsim_shaper_set, 510 .delete = nsim_shaper_del, 511 .group = nsim_shaper_group, 512 .capabilities = nsim_shaper_cap, 513 }; 514 515 static const struct net_device_ops nsim_netdev_ops = { 516 .ndo_start_xmit = nsim_start_xmit, 517 .ndo_set_rx_mode = nsim_set_rx_mode, 518 .ndo_set_mac_address = eth_mac_addr, 519 .ndo_validate_addr = eth_validate_addr, 520 .ndo_change_mtu = nsim_change_mtu, 521 .ndo_get_stats64 = nsim_get_stats64, 522 .ndo_set_vf_mac = nsim_set_vf_mac, 523 .ndo_set_vf_vlan = nsim_set_vf_vlan, 524 .ndo_set_vf_rate = nsim_set_vf_rate, 525 .ndo_set_vf_spoofchk = nsim_set_vf_spoofchk, 526 .ndo_set_vf_trust = nsim_set_vf_trust, 527 .ndo_get_vf_config = nsim_get_vf_config, 528 .ndo_set_vf_link_state = nsim_set_vf_link_state, 529 .ndo_set_vf_rss_query_en = nsim_set_vf_rss_query_en, 530 .ndo_setup_tc = nsim_setup_tc, 531 .ndo_set_features = nsim_set_features, 532 .ndo_get_iflink = nsim_get_iflink, 533 .ndo_bpf = nsim_bpf, 534 .ndo_open = nsim_open, 535 .ndo_stop = nsim_stop, 536 .net_shaper_ops = &nsim_shaper_ops, 537 }; 538 539 static const struct net_device_ops nsim_vf_netdev_ops = { 540 .ndo_start_xmit = nsim_start_xmit, 541 .ndo_set_rx_mode = nsim_set_rx_mode, 542 .ndo_set_mac_address = eth_mac_addr, 543 .ndo_validate_addr = eth_validate_addr, 544 .ndo_change_mtu = nsim_change_mtu, 545 .ndo_get_stats64 = nsim_get_stats64, 546 .ndo_setup_tc = nsim_setup_tc, 547 .ndo_set_features = nsim_set_features, 548 }; 549 550 /* We don't have true per-queue stats, yet, so do some random fakery here. 551 * Only report stuff for queue 0. 552 */ 553 static void nsim_get_queue_stats_rx(struct net_device *dev, int idx, 554 struct netdev_queue_stats_rx *stats) 555 { 556 struct rtnl_link_stats64 rtstats = {}; 557 558 if (!idx) 559 nsim_get_stats64(dev, &rtstats); 560 561 stats->packets = rtstats.rx_packets - !!rtstats.rx_packets; 562 stats->bytes = rtstats.rx_bytes; 563 } 564 565 static void nsim_get_queue_stats_tx(struct net_device *dev, int idx, 566 struct netdev_queue_stats_tx *stats) 567 { 568 struct rtnl_link_stats64 rtstats = {}; 569 570 if (!idx) 571 nsim_get_stats64(dev, &rtstats); 572 573 stats->packets = rtstats.tx_packets - !!rtstats.tx_packets; 574 stats->bytes = rtstats.tx_bytes; 575 } 576 577 static void nsim_get_base_stats(struct net_device *dev, 578 struct netdev_queue_stats_rx *rx, 579 struct netdev_queue_stats_tx *tx) 580 { 581 struct rtnl_link_stats64 rtstats = {}; 582 583 nsim_get_stats64(dev, &rtstats); 584 585 rx->packets = !!rtstats.rx_packets; 586 rx->bytes = 0; 587 tx->packets = !!rtstats.tx_packets; 588 tx->bytes = 0; 589 } 590 591 static const struct netdev_stat_ops nsim_stat_ops = { 592 .get_queue_stats_tx = nsim_get_queue_stats_tx, 593 .get_queue_stats_rx = nsim_get_queue_stats_rx, 594 .get_base_stats = nsim_get_base_stats, 595 }; 596 597 static struct nsim_rq *nsim_queue_alloc(void) 598 { 599 struct nsim_rq *rq; 600 601 rq = kzalloc(sizeof(*rq), GFP_KERNEL_ACCOUNT); 602 if (!rq) 603 return NULL; 604 605 skb_queue_head_init(&rq->skb_queue); 606 return rq; 607 } 608 609 static void nsim_queue_free(struct nsim_rq *rq) 610 { 611 skb_queue_purge_reason(&rq->skb_queue, SKB_DROP_REASON_QUEUE_PURGE); 612 kfree(rq); 613 } 614 615 /* Queue reset mode is controlled by ns->rq_reset_mode. 616 * - normal - new NAPI new pool (old NAPI enabled when new added) 617 * - mode 1 - allocate new pool (NAPI is only disabled / enabled) 618 * - mode 2 - new NAPI new pool (old NAPI removed before new added) 619 * - mode 3 - new NAPI new pool (old NAPI disabled when new added) 620 */ 621 struct nsim_queue_mem { 622 struct nsim_rq *rq; 623 struct page_pool *pp; 624 }; 625 626 static int 627 nsim_queue_mem_alloc(struct net_device *dev, void *per_queue_mem, int idx) 628 { 629 struct nsim_queue_mem *qmem = per_queue_mem; 630 struct netdevsim *ns = netdev_priv(dev); 631 int err; 632 633 if (ns->rq_reset_mode > 3) 634 return -EINVAL; 635 636 if (ns->rq_reset_mode == 1) 637 return nsim_create_page_pool(&qmem->pp, &ns->rq[idx]->napi); 638 639 qmem->rq = nsim_queue_alloc(); 640 if (!qmem->rq) 641 return -ENOMEM; 642 643 err = nsim_create_page_pool(&qmem->rq->page_pool, &qmem->rq->napi); 644 if (err) 645 goto err_free; 646 647 if (!ns->rq_reset_mode) 648 netif_napi_add_config(dev, &qmem->rq->napi, nsim_poll, idx); 649 650 return 0; 651 652 err_free: 653 nsim_queue_free(qmem->rq); 654 return err; 655 } 656 657 static void nsim_queue_mem_free(struct net_device *dev, void *per_queue_mem) 658 { 659 struct nsim_queue_mem *qmem = per_queue_mem; 660 struct netdevsim *ns = netdev_priv(dev); 661 662 page_pool_destroy(qmem->pp); 663 if (qmem->rq) { 664 if (!ns->rq_reset_mode) 665 netif_napi_del(&qmem->rq->napi); 666 page_pool_destroy(qmem->rq->page_pool); 667 nsim_queue_free(qmem->rq); 668 } 669 } 670 671 static int 672 nsim_queue_start(struct net_device *dev, void *per_queue_mem, int idx) 673 { 674 struct nsim_queue_mem *qmem = per_queue_mem; 675 struct netdevsim *ns = netdev_priv(dev); 676 677 if (ns->rq_reset_mode == 1) { 678 ns->rq[idx]->page_pool = qmem->pp; 679 napi_enable(&ns->rq[idx]->napi); 680 return 0; 681 } 682 683 /* netif_napi_add()/_del() should normally be called from alloc/free, 684 * here we want to test various call orders. 685 */ 686 if (ns->rq_reset_mode == 2) { 687 netif_napi_del(&ns->rq[idx]->napi); 688 netif_napi_add_config(dev, &qmem->rq->napi, nsim_poll, idx); 689 } else if (ns->rq_reset_mode == 3) { 690 netif_napi_add_config(dev, &qmem->rq->napi, nsim_poll, idx); 691 netif_napi_del(&ns->rq[idx]->napi); 692 } 693 694 ns->rq[idx] = qmem->rq; 695 napi_enable(&ns->rq[idx]->napi); 696 697 return 0; 698 } 699 700 static int nsim_queue_stop(struct net_device *dev, void *per_queue_mem, int idx) 701 { 702 struct nsim_queue_mem *qmem = per_queue_mem; 703 struct netdevsim *ns = netdev_priv(dev); 704 705 napi_disable(&ns->rq[idx]->napi); 706 707 if (ns->rq_reset_mode == 1) { 708 qmem->pp = ns->rq[idx]->page_pool; 709 page_pool_disable_direct_recycling(qmem->pp); 710 } else { 711 qmem->rq = ns->rq[idx]; 712 } 713 714 return 0; 715 } 716 717 static const struct netdev_queue_mgmt_ops nsim_queue_mgmt_ops = { 718 .ndo_queue_mem_size = sizeof(struct nsim_queue_mem), 719 .ndo_queue_mem_alloc = nsim_queue_mem_alloc, 720 .ndo_queue_mem_free = nsim_queue_mem_free, 721 .ndo_queue_start = nsim_queue_start, 722 .ndo_queue_stop = nsim_queue_stop, 723 }; 724 725 static ssize_t 726 nsim_pp_hold_read(struct file *file, char __user *data, 727 size_t count, loff_t *ppos) 728 { 729 struct netdevsim *ns = file->private_data; 730 char buf[3] = "n\n"; 731 732 if (ns->page) 733 buf[0] = 'y'; 734 735 return simple_read_from_buffer(data, count, ppos, buf, 2); 736 } 737 738 static ssize_t 739 nsim_pp_hold_write(struct file *file, const char __user *data, 740 size_t count, loff_t *ppos) 741 { 742 struct netdevsim *ns = file->private_data; 743 ssize_t ret; 744 bool val; 745 746 ret = kstrtobool_from_user(data, count, &val); 747 if (ret) 748 return ret; 749 750 rtnl_lock(); 751 ret = count; 752 if (val == !!ns->page) 753 goto exit; 754 755 if (!netif_running(ns->netdev) && val) { 756 ret = -ENETDOWN; 757 } else if (val) { 758 ns->page = page_pool_dev_alloc_pages(ns->rq[0]->page_pool); 759 if (!ns->page) 760 ret = -ENOMEM; 761 } else { 762 page_pool_put_full_page(ns->page->pp, ns->page, false); 763 ns->page = NULL; 764 } 765 766 exit: 767 rtnl_unlock(); 768 return ret; 769 } 770 771 static const struct file_operations nsim_pp_hold_fops = { 772 .open = simple_open, 773 .read = nsim_pp_hold_read, 774 .write = nsim_pp_hold_write, 775 .llseek = generic_file_llseek, 776 .owner = THIS_MODULE, 777 }; 778 779 static void nsim_setup(struct net_device *dev) 780 { 781 ether_setup(dev); 782 eth_hw_addr_random(dev); 783 784 dev->tx_queue_len = 0; 785 dev->flags &= ~IFF_MULTICAST; 786 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE | 787 IFF_NO_QUEUE; 788 dev->features |= NETIF_F_HIGHDMA | 789 NETIF_F_SG | 790 NETIF_F_FRAGLIST | 791 NETIF_F_HW_CSUM | 792 NETIF_F_TSO; 793 dev->hw_features |= NETIF_F_HW_TC | 794 NETIF_F_SG | 795 NETIF_F_FRAGLIST | 796 NETIF_F_HW_CSUM | 797 NETIF_F_TSO; 798 dev->max_mtu = ETH_MAX_MTU; 799 dev->xdp_features = NETDEV_XDP_ACT_HW_OFFLOAD; 800 } 801 802 static int nsim_queue_init(struct netdevsim *ns) 803 { 804 struct net_device *dev = ns->netdev; 805 int i; 806 807 ns->rq = kcalloc(dev->num_rx_queues, sizeof(*ns->rq), 808 GFP_KERNEL_ACCOUNT); 809 if (!ns->rq) 810 return -ENOMEM; 811 812 for (i = 0; i < dev->num_rx_queues; i++) { 813 ns->rq[i] = nsim_queue_alloc(); 814 if (!ns->rq[i]) 815 goto err_free_prev; 816 } 817 818 return 0; 819 820 err_free_prev: 821 while (i--) 822 kfree(ns->rq[i]); 823 kfree(ns->rq); 824 return -ENOMEM; 825 } 826 827 static void nsim_queue_uninit(struct netdevsim *ns) 828 { 829 struct net_device *dev = ns->netdev; 830 int i; 831 832 for (i = 0; i < dev->num_rx_queues; i++) 833 nsim_queue_free(ns->rq[i]); 834 835 kfree(ns->rq); 836 ns->rq = NULL; 837 } 838 839 static int nsim_init_netdevsim(struct netdevsim *ns) 840 { 841 struct mock_phc *phc; 842 int err; 843 844 phc = mock_phc_create(&ns->nsim_bus_dev->dev); 845 if (IS_ERR(phc)) 846 return PTR_ERR(phc); 847 848 ns->phc = phc; 849 ns->netdev->netdev_ops = &nsim_netdev_ops; 850 ns->netdev->stat_ops = &nsim_stat_ops; 851 ns->netdev->queue_mgmt_ops = &nsim_queue_mgmt_ops; 852 853 err = nsim_udp_tunnels_info_create(ns->nsim_dev, ns->netdev); 854 if (err) 855 goto err_phc_destroy; 856 857 rtnl_lock(); 858 err = nsim_queue_init(ns); 859 if (err) 860 goto err_utn_destroy; 861 862 err = nsim_bpf_init(ns); 863 if (err) 864 goto err_rq_destroy; 865 866 nsim_macsec_init(ns); 867 nsim_ipsec_init(ns); 868 869 err = register_netdevice(ns->netdev); 870 if (err) 871 goto err_ipsec_teardown; 872 rtnl_unlock(); 873 return 0; 874 875 err_ipsec_teardown: 876 nsim_ipsec_teardown(ns); 877 nsim_macsec_teardown(ns); 878 nsim_bpf_uninit(ns); 879 err_rq_destroy: 880 nsim_queue_uninit(ns); 881 err_utn_destroy: 882 rtnl_unlock(); 883 nsim_udp_tunnels_info_destroy(ns->netdev); 884 err_phc_destroy: 885 mock_phc_destroy(ns->phc); 886 return err; 887 } 888 889 static int nsim_init_netdevsim_vf(struct netdevsim *ns) 890 { 891 int err; 892 893 ns->netdev->netdev_ops = &nsim_vf_netdev_ops; 894 rtnl_lock(); 895 err = register_netdevice(ns->netdev); 896 rtnl_unlock(); 897 return err; 898 } 899 900 static void nsim_exit_netdevsim(struct netdevsim *ns) 901 { 902 nsim_udp_tunnels_info_destroy(ns->netdev); 903 mock_phc_destroy(ns->phc); 904 } 905 906 struct netdevsim * 907 nsim_create(struct nsim_dev *nsim_dev, struct nsim_dev_port *nsim_dev_port) 908 { 909 struct net_device *dev; 910 struct netdevsim *ns; 911 int err; 912 913 dev = alloc_netdev_mq(sizeof(*ns), "eth%d", NET_NAME_UNKNOWN, nsim_setup, 914 nsim_dev->nsim_bus_dev->num_queues); 915 if (!dev) 916 return ERR_PTR(-ENOMEM); 917 918 dev_net_set(dev, nsim_dev_net(nsim_dev)); 919 ns = netdev_priv(dev); 920 ns->netdev = dev; 921 u64_stats_init(&ns->syncp); 922 ns->nsim_dev = nsim_dev; 923 ns->nsim_dev_port = nsim_dev_port; 924 ns->nsim_bus_dev = nsim_dev->nsim_bus_dev; 925 SET_NETDEV_DEV(dev, &ns->nsim_bus_dev->dev); 926 SET_NETDEV_DEVLINK_PORT(dev, &nsim_dev_port->devlink_port); 927 nsim_ethtool_init(ns); 928 if (nsim_dev_port_is_pf(nsim_dev_port)) 929 err = nsim_init_netdevsim(ns); 930 else 931 err = nsim_init_netdevsim_vf(ns); 932 if (err) 933 goto err_free_netdev; 934 935 ns->pp_dfs = debugfs_create_file("pp_hold", 0600, nsim_dev_port->ddir, 936 ns, &nsim_pp_hold_fops); 937 938 return ns; 939 940 err_free_netdev: 941 free_netdev(dev); 942 return ERR_PTR(err); 943 } 944 945 void nsim_destroy(struct netdevsim *ns) 946 { 947 struct net_device *dev = ns->netdev; 948 struct netdevsim *peer; 949 950 debugfs_remove(ns->pp_dfs); 951 952 rtnl_lock(); 953 peer = rtnl_dereference(ns->peer); 954 if (peer) 955 RCU_INIT_POINTER(peer->peer, NULL); 956 RCU_INIT_POINTER(ns->peer, NULL); 957 unregister_netdevice(dev); 958 if (nsim_dev_port_is_pf(ns->nsim_dev_port)) { 959 nsim_macsec_teardown(ns); 960 nsim_ipsec_teardown(ns); 961 nsim_bpf_uninit(ns); 962 nsim_queue_uninit(ns); 963 } 964 rtnl_unlock(); 965 if (nsim_dev_port_is_pf(ns->nsim_dev_port)) 966 nsim_exit_netdevsim(ns); 967 968 /* Put this intentionally late to exercise the orphaning path */ 969 if (ns->page) { 970 page_pool_put_full_page(ns->page->pp, ns->page, false); 971 ns->page = NULL; 972 } 973 974 free_netdev(dev); 975 } 976 977 bool netdev_is_nsim(struct net_device *dev) 978 { 979 return dev->netdev_ops == &nsim_netdev_ops; 980 } 981 982 static int nsim_validate(struct nlattr *tb[], struct nlattr *data[], 983 struct netlink_ext_ack *extack) 984 { 985 NL_SET_ERR_MSG_MOD(extack, 986 "Please use: echo \"[ID] [PORT_COUNT] [NUM_QUEUES]\" > /sys/bus/netdevsim/new_device"); 987 return -EOPNOTSUPP; 988 } 989 990 static struct rtnl_link_ops nsim_link_ops __read_mostly = { 991 .kind = DRV_NAME, 992 .validate = nsim_validate, 993 }; 994 995 static int __init nsim_module_init(void) 996 { 997 int err; 998 999 err = nsim_dev_init(); 1000 if (err) 1001 return err; 1002 1003 err = nsim_bus_init(); 1004 if (err) 1005 goto err_dev_exit; 1006 1007 err = rtnl_link_register(&nsim_link_ops); 1008 if (err) 1009 goto err_bus_exit; 1010 1011 return 0; 1012 1013 err_bus_exit: 1014 nsim_bus_exit(); 1015 err_dev_exit: 1016 nsim_dev_exit(); 1017 return err; 1018 } 1019 1020 static void __exit nsim_module_exit(void) 1021 { 1022 rtnl_link_unregister(&nsim_link_ops); 1023 nsim_bus_exit(); 1024 nsim_dev_exit(); 1025 } 1026 1027 module_init(nsim_module_init); 1028 module_exit(nsim_module_exit); 1029 MODULE_LICENSE("GPL"); 1030 MODULE_DESCRIPTION("Simulated networking device for testing"); 1031 MODULE_ALIAS_RTNL_LINK(DRV_NAME); 1032