1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright(c) 2010-2014 Intel Corporation. 4 */ 5 6 #include <linux/version.h> 7 #include <linux/module.h> 8 #include <linux/miscdevice.h> 9 #include <linux/netdevice.h> 10 #include <linux/etherdevice.h> 11 #include <linux/pci.h> 12 #include <linux/kthread.h> 13 #include <linux/rwsem.h> 14 #include <linux/mutex.h> 15 #include <linux/nsproxy.h> 16 #include <net/net_namespace.h> 17 #include <net/netns/generic.h> 18 19 #include <exec-env/rte_kni_common.h> 20 21 #include "compat.h" 22 #include "kni_dev.h" 23 24 MODULE_LICENSE("Dual BSD/GPL"); 25 MODULE_AUTHOR("Intel Corporation"); 26 MODULE_DESCRIPTION("Kernel Module for managing kni devices"); 27 28 #define KNI_RX_LOOP_NUM 1000 29 30 #define KNI_MAX_DEVICES 32 31 32 extern const struct pci_device_id ixgbe_pci_tbl[]; 33 extern const struct pci_device_id igb_pci_tbl[]; 34 35 /* loopback mode */ 36 static char *lo_mode; 37 38 /* Kernel thread mode */ 39 static char *kthread_mode; 40 static uint32_t multiple_kthread_on; 41 42 #define KNI_DEV_IN_USE_BIT_NUM 0 /* Bit number for device in use */ 43 44 static int kni_net_id; 45 46 struct kni_net { 47 unsigned long device_in_use; /* device in use flag */ 48 struct mutex kni_kthread_lock; 49 struct task_struct *kni_kthread; 50 struct rw_semaphore kni_list_lock; 51 struct list_head kni_list_head; 52 }; 53 54 static int __net_init 55 kni_init_net(struct net *net) 56 { 57 #ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS 58 struct kni_net *knet = net_generic(net, kni_net_id); 59 60 memset(knet, 0, sizeof(*knet)); 61 #else 62 struct kni_net *knet; 63 int ret; 64 65 knet = kzalloc(sizeof(struct kni_net), GFP_KERNEL); 66 if (!knet) { 67 ret = -ENOMEM; 68 return ret; 69 } 70 #endif 71 72 /* Clear the bit of device in use */ 73 clear_bit(KNI_DEV_IN_USE_BIT_NUM, &knet->device_in_use); 74 75 mutex_init(&knet->kni_kthread_lock); 76 77 init_rwsem(&knet->kni_list_lock); 78 INIT_LIST_HEAD(&knet->kni_list_head); 79 80 #ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS 81 return 0; 82 #else 83 ret = net_assign_generic(net, kni_net_id, knet); 84 if (ret < 0) 85 kfree(knet); 86 87 return ret; 88 #endif 89 } 90 91 static void __net_exit 92 kni_exit_net(struct net *net) 93 { 94 struct kni_net *knet __maybe_unused; 95 96 knet = net_generic(net, kni_net_id); 97 mutex_destroy(&knet->kni_kthread_lock); 98 99 #ifndef HAVE_SIMPLIFIED_PERNET_OPERATIONS 100 kfree(knet); 101 #endif 102 } 103 104 static struct pernet_operations kni_net_ops = { 105 .init = kni_init_net, 106 .exit = kni_exit_net, 107 #ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS 108 .id = &kni_net_id, 109 .size = sizeof(struct kni_net), 110 #endif 111 }; 112 113 static int 114 kni_thread_single(void *data) 115 { 116 struct kni_net *knet = data; 117 int j; 118 struct kni_dev *dev; 119 120 while (!kthread_should_stop()) { 121 down_read(&knet->kni_list_lock); 122 for (j = 0; j < KNI_RX_LOOP_NUM; j++) { 123 list_for_each_entry(dev, &knet->kni_list_head, list) { 124 kni_net_rx(dev); 125 kni_net_poll_resp(dev); 126 } 127 } 128 up_read(&knet->kni_list_lock); 129 #ifdef RTE_KNI_PREEMPT_DEFAULT 130 /* reschedule out for a while */ 131 schedule_timeout_interruptible( 132 usecs_to_jiffies(KNI_KTHREAD_RESCHEDULE_INTERVAL)); 133 #endif 134 } 135 136 return 0; 137 } 138 139 static int 140 kni_thread_multiple(void *param) 141 { 142 int j; 143 struct kni_dev *dev = param; 144 145 while (!kthread_should_stop()) { 146 for (j = 0; j < KNI_RX_LOOP_NUM; j++) { 147 kni_net_rx(dev); 148 kni_net_poll_resp(dev); 149 } 150 #ifdef RTE_KNI_PREEMPT_DEFAULT 151 schedule_timeout_interruptible( 152 usecs_to_jiffies(KNI_KTHREAD_RESCHEDULE_INTERVAL)); 153 #endif 154 } 155 156 return 0; 157 } 158 159 static int 160 kni_open(struct inode *inode, struct file *file) 161 { 162 struct net *net = current->nsproxy->net_ns; 163 struct kni_net *knet = net_generic(net, kni_net_id); 164 165 /* kni device can be opened by one user only per netns */ 166 if (test_and_set_bit(KNI_DEV_IN_USE_BIT_NUM, &knet->device_in_use)) 167 return -EBUSY; 168 169 file->private_data = get_net(net); 170 pr_debug("/dev/kni opened\n"); 171 172 return 0; 173 } 174 175 static int 176 kni_dev_remove(struct kni_dev *dev) 177 { 178 if (!dev) 179 return -ENODEV; 180 181 #ifdef RTE_KNI_KMOD_ETHTOOL 182 if (dev->pci_dev) { 183 if (pci_match_id(ixgbe_pci_tbl, dev->pci_dev)) 184 ixgbe_kni_remove(dev->pci_dev); 185 else if (pci_match_id(igb_pci_tbl, dev->pci_dev)) 186 igb_kni_remove(dev->pci_dev); 187 } 188 #endif 189 190 if (dev->net_dev) { 191 unregister_netdev(dev->net_dev); 192 free_netdev(dev->net_dev); 193 } 194 195 kni_net_release_fifo_phy(dev); 196 197 return 0; 198 } 199 200 static int 201 kni_release(struct inode *inode, struct file *file) 202 { 203 struct net *net = file->private_data; 204 struct kni_net *knet = net_generic(net, kni_net_id); 205 struct kni_dev *dev, *n; 206 207 /* Stop kernel thread for single mode */ 208 if (multiple_kthread_on == 0) { 209 mutex_lock(&knet->kni_kthread_lock); 210 /* Stop kernel thread */ 211 if (knet->kni_kthread != NULL) { 212 kthread_stop(knet->kni_kthread); 213 knet->kni_kthread = NULL; 214 } 215 mutex_unlock(&knet->kni_kthread_lock); 216 } 217 218 down_write(&knet->kni_list_lock); 219 list_for_each_entry_safe(dev, n, &knet->kni_list_head, list) { 220 /* Stop kernel thread for multiple mode */ 221 if (multiple_kthread_on && dev->pthread != NULL) { 222 kthread_stop(dev->pthread); 223 dev->pthread = NULL; 224 } 225 226 kni_dev_remove(dev); 227 list_del(&dev->list); 228 } 229 up_write(&knet->kni_list_lock); 230 231 /* Clear the bit of device in use */ 232 clear_bit(KNI_DEV_IN_USE_BIT_NUM, &knet->device_in_use); 233 234 put_net(net); 235 pr_debug("/dev/kni closed\n"); 236 237 return 0; 238 } 239 240 static int 241 kni_check_param(struct kni_dev *kni, struct rte_kni_device_info *dev) 242 { 243 if (!kni || !dev) 244 return -1; 245 246 /* Check if network name has been used */ 247 if (!strncmp(kni->name, dev->name, RTE_KNI_NAMESIZE)) { 248 pr_err("KNI name %s duplicated\n", dev->name); 249 return -1; 250 } 251 252 return 0; 253 } 254 255 static int 256 kni_run_thread(struct kni_net *knet, struct kni_dev *kni, uint8_t force_bind) 257 { 258 /** 259 * Create a new kernel thread for multiple mode, set its core affinity, 260 * and finally wake it up. 261 */ 262 if (multiple_kthread_on) { 263 kni->pthread = kthread_create(kni_thread_multiple, 264 (void *)kni, "kni_%s", kni->name); 265 if (IS_ERR(kni->pthread)) { 266 kni_dev_remove(kni); 267 return -ECANCELED; 268 } 269 270 if (force_bind) 271 kthread_bind(kni->pthread, kni->core_id); 272 wake_up_process(kni->pthread); 273 } else { 274 mutex_lock(&knet->kni_kthread_lock); 275 276 if (knet->kni_kthread == NULL) { 277 knet->kni_kthread = kthread_create(kni_thread_single, 278 (void *)knet, "kni_single"); 279 if (IS_ERR(knet->kni_kthread)) { 280 mutex_unlock(&knet->kni_kthread_lock); 281 kni_dev_remove(kni); 282 return -ECANCELED; 283 } 284 285 if (force_bind) 286 kthread_bind(knet->kni_kthread, kni->core_id); 287 wake_up_process(knet->kni_kthread); 288 } 289 290 mutex_unlock(&knet->kni_kthread_lock); 291 } 292 293 return 0; 294 } 295 296 static int 297 kni_ioctl_create(struct net *net, uint32_t ioctl_num, 298 unsigned long ioctl_param) 299 { 300 struct kni_net *knet = net_generic(net, kni_net_id); 301 int ret; 302 struct rte_kni_device_info dev_info; 303 struct net_device *net_dev = NULL; 304 struct kni_dev *kni, *dev, *n; 305 #ifdef RTE_KNI_KMOD_ETHTOOL 306 struct pci_dev *found_pci = NULL; 307 struct net_device *lad_dev = NULL; 308 struct pci_dev *pci = NULL; 309 #endif 310 311 pr_info("Creating kni...\n"); 312 /* Check the buffer size, to avoid warning */ 313 if (_IOC_SIZE(ioctl_num) > sizeof(dev_info)) 314 return -EINVAL; 315 316 /* Copy kni info from user space */ 317 ret = copy_from_user(&dev_info, (void *)ioctl_param, sizeof(dev_info)); 318 if (ret) { 319 pr_err("copy_from_user in kni_ioctl_create"); 320 return -EIO; 321 } 322 323 /* Check if name is zero-ended */ 324 if (strnlen(dev_info.name, sizeof(dev_info.name)) == sizeof(dev_info.name)) { 325 pr_err("kni.name not zero-terminated"); 326 return -EINVAL; 327 } 328 329 /** 330 * Check if the cpu core id is valid for binding. 331 */ 332 if (dev_info.force_bind && !cpu_online(dev_info.core_id)) { 333 pr_err("cpu %u is not online\n", dev_info.core_id); 334 return -EINVAL; 335 } 336 337 /* Check if it has been created */ 338 down_read(&knet->kni_list_lock); 339 list_for_each_entry_safe(dev, n, &knet->kni_list_head, list) { 340 if (kni_check_param(dev, &dev_info) < 0) { 341 up_read(&knet->kni_list_lock); 342 return -EINVAL; 343 } 344 } 345 up_read(&knet->kni_list_lock); 346 347 net_dev = alloc_netdev(sizeof(struct kni_dev), dev_info.name, 348 #ifdef NET_NAME_USER 349 NET_NAME_USER, 350 #endif 351 kni_net_init); 352 if (net_dev == NULL) { 353 pr_err("error allocating device \"%s\"\n", dev_info.name); 354 return -EBUSY; 355 } 356 357 dev_net_set(net_dev, net); 358 359 kni = netdev_priv(net_dev); 360 361 kni->net_dev = net_dev; 362 kni->group_id = dev_info.group_id; 363 kni->core_id = dev_info.core_id; 364 strncpy(kni->name, dev_info.name, RTE_KNI_NAMESIZE); 365 366 /* Translate user space info into kernel space info */ 367 kni->tx_q = phys_to_virt(dev_info.tx_phys); 368 kni->rx_q = phys_to_virt(dev_info.rx_phys); 369 kni->alloc_q = phys_to_virt(dev_info.alloc_phys); 370 kni->free_q = phys_to_virt(dev_info.free_phys); 371 372 kni->req_q = phys_to_virt(dev_info.req_phys); 373 kni->resp_q = phys_to_virt(dev_info.resp_phys); 374 kni->sync_va = dev_info.sync_va; 375 kni->sync_kva = phys_to_virt(dev_info.sync_phys); 376 377 kni->mbuf_size = dev_info.mbuf_size; 378 379 pr_debug("tx_phys: 0x%016llx, tx_q addr: 0x%p\n", 380 (unsigned long long) dev_info.tx_phys, kni->tx_q); 381 pr_debug("rx_phys: 0x%016llx, rx_q addr: 0x%p\n", 382 (unsigned long long) dev_info.rx_phys, kni->rx_q); 383 pr_debug("alloc_phys: 0x%016llx, alloc_q addr: 0x%p\n", 384 (unsigned long long) dev_info.alloc_phys, kni->alloc_q); 385 pr_debug("free_phys: 0x%016llx, free_q addr: 0x%p\n", 386 (unsigned long long) dev_info.free_phys, kni->free_q); 387 pr_debug("req_phys: 0x%016llx, req_q addr: 0x%p\n", 388 (unsigned long long) dev_info.req_phys, kni->req_q); 389 pr_debug("resp_phys: 0x%016llx, resp_q addr: 0x%p\n", 390 (unsigned long long) dev_info.resp_phys, kni->resp_q); 391 pr_debug("mbuf_size: %u\n", kni->mbuf_size); 392 393 pr_debug("PCI: %02x:%02x.%02x %04x:%04x\n", 394 dev_info.bus, 395 dev_info.devid, 396 dev_info.function, 397 dev_info.vendor_id, 398 dev_info.device_id); 399 #ifdef RTE_KNI_KMOD_ETHTOOL 400 pci = pci_get_device(dev_info.vendor_id, dev_info.device_id, NULL); 401 402 /* Support Ethtool */ 403 while (pci) { 404 pr_debug("pci_bus: %02x:%02x:%02x\n", 405 pci->bus->number, 406 PCI_SLOT(pci->devfn), 407 PCI_FUNC(pci->devfn)); 408 409 if ((pci->bus->number == dev_info.bus) && 410 (PCI_SLOT(pci->devfn) == dev_info.devid) && 411 (PCI_FUNC(pci->devfn) == dev_info.function)) { 412 found_pci = pci; 413 414 if (pci_match_id(ixgbe_pci_tbl, found_pci)) 415 ret = ixgbe_kni_probe(found_pci, &lad_dev); 416 else if (pci_match_id(igb_pci_tbl, found_pci)) 417 ret = igb_kni_probe(found_pci, &lad_dev); 418 else 419 ret = -1; 420 421 pr_debug("PCI found: pci=0x%p, lad_dev=0x%p\n", 422 pci, lad_dev); 423 if (ret == 0) { 424 kni->lad_dev = lad_dev; 425 kni_set_ethtool_ops(kni->net_dev); 426 } else { 427 pr_err("Device not supported by ethtool"); 428 kni->lad_dev = NULL; 429 } 430 431 kni->pci_dev = found_pci; 432 kni->device_id = dev_info.device_id; 433 break; 434 } 435 pci = pci_get_device(dev_info.vendor_id, 436 dev_info.device_id, pci); 437 } 438 if (pci) 439 pci_dev_put(pci); 440 #endif 441 442 if (kni->lad_dev) 443 ether_addr_copy(net_dev->dev_addr, kni->lad_dev->dev_addr); 444 else { 445 /* if user has provided a valid mac address */ 446 if (is_valid_ether_addr((unsigned char *)(dev_info.mac_addr))) 447 memcpy(net_dev->dev_addr, dev_info.mac_addr, ETH_ALEN); 448 else 449 /* 450 * Generate random mac address. eth_random_addr() is the 451 * newer version of generating mac address in kernel. 452 */ 453 random_ether_addr(net_dev->dev_addr); 454 } 455 456 if (dev_info.mtu) 457 net_dev->mtu = dev_info.mtu; 458 459 ret = register_netdev(net_dev); 460 if (ret) { 461 pr_err("error %i registering device \"%s\"\n", 462 ret, dev_info.name); 463 kni->net_dev = NULL; 464 kni_dev_remove(kni); 465 free_netdev(net_dev); 466 return -ENODEV; 467 } 468 469 ret = kni_run_thread(knet, kni, dev_info.force_bind); 470 if (ret != 0) 471 return ret; 472 473 down_write(&knet->kni_list_lock); 474 list_add(&kni->list, &knet->kni_list_head); 475 up_write(&knet->kni_list_lock); 476 477 return 0; 478 } 479 480 static int 481 kni_ioctl_release(struct net *net, uint32_t ioctl_num, 482 unsigned long ioctl_param) 483 { 484 struct kni_net *knet = net_generic(net, kni_net_id); 485 int ret = -EINVAL; 486 struct kni_dev *dev, *n; 487 struct rte_kni_device_info dev_info; 488 489 if (_IOC_SIZE(ioctl_num) > sizeof(dev_info)) 490 return -EINVAL; 491 492 ret = copy_from_user(&dev_info, (void *)ioctl_param, sizeof(dev_info)); 493 if (ret) { 494 pr_err("copy_from_user in kni_ioctl_release"); 495 return -EIO; 496 } 497 498 /* Release the network device according to its name */ 499 if (strlen(dev_info.name) == 0) 500 return ret; 501 502 down_write(&knet->kni_list_lock); 503 list_for_each_entry_safe(dev, n, &knet->kni_list_head, list) { 504 if (strncmp(dev->name, dev_info.name, RTE_KNI_NAMESIZE) != 0) 505 continue; 506 507 if (multiple_kthread_on && dev->pthread != NULL) { 508 kthread_stop(dev->pthread); 509 dev->pthread = NULL; 510 } 511 512 kni_dev_remove(dev); 513 list_del(&dev->list); 514 ret = 0; 515 break; 516 } 517 up_write(&knet->kni_list_lock); 518 pr_info("%s release kni named %s\n", 519 (ret == 0 ? "Successfully" : "Unsuccessfully"), dev_info.name); 520 521 return ret; 522 } 523 524 static int 525 kni_ioctl(struct inode *inode, uint32_t ioctl_num, unsigned long ioctl_param) 526 { 527 int ret = -EINVAL; 528 struct net *net = current->nsproxy->net_ns; 529 530 pr_debug("IOCTL num=0x%0x param=0x%0lx\n", ioctl_num, ioctl_param); 531 532 /* 533 * Switch according to the ioctl called 534 */ 535 switch (_IOC_NR(ioctl_num)) { 536 case _IOC_NR(RTE_KNI_IOCTL_TEST): 537 /* For test only, not used */ 538 break; 539 case _IOC_NR(RTE_KNI_IOCTL_CREATE): 540 ret = kni_ioctl_create(net, ioctl_num, ioctl_param); 541 break; 542 case _IOC_NR(RTE_KNI_IOCTL_RELEASE): 543 ret = kni_ioctl_release(net, ioctl_num, ioctl_param); 544 break; 545 default: 546 pr_debug("IOCTL default\n"); 547 break; 548 } 549 550 return ret; 551 } 552 553 static int 554 kni_compat_ioctl(struct inode *inode, uint32_t ioctl_num, 555 unsigned long ioctl_param) 556 { 557 /* 32 bits app on 64 bits OS to be supported later */ 558 pr_debug("Not implemented.\n"); 559 560 return -EINVAL; 561 } 562 563 static const struct file_operations kni_fops = { 564 .owner = THIS_MODULE, 565 .open = kni_open, 566 .release = kni_release, 567 .unlocked_ioctl = (void *)kni_ioctl, 568 .compat_ioctl = (void *)kni_compat_ioctl, 569 }; 570 571 static struct miscdevice kni_misc = { 572 .minor = MISC_DYNAMIC_MINOR, 573 .name = KNI_DEVICE, 574 .fops = &kni_fops, 575 }; 576 577 static int __init 578 kni_parse_kthread_mode(void) 579 { 580 if (!kthread_mode) 581 return 0; 582 583 if (strcmp(kthread_mode, "single") == 0) 584 return 0; 585 else if (strcmp(kthread_mode, "multiple") == 0) 586 multiple_kthread_on = 1; 587 else 588 return -1; 589 590 return 0; 591 } 592 593 static int __init 594 kni_init(void) 595 { 596 int rc; 597 598 if (kni_parse_kthread_mode() < 0) { 599 pr_err("Invalid parameter for kthread_mode\n"); 600 return -EINVAL; 601 } 602 603 if (multiple_kthread_on == 0) 604 pr_debug("Single kernel thread for all KNI devices\n"); 605 else 606 pr_debug("Multiple kernel thread mode enabled\n"); 607 608 #ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS 609 rc = register_pernet_subsys(&kni_net_ops); 610 #else 611 rc = register_pernet_gen_subsys(&kni_net_id, &kni_net_ops); 612 #endif 613 if (rc) 614 return -EPERM; 615 616 rc = misc_register(&kni_misc); 617 if (rc != 0) { 618 pr_err("Misc registration failed\n"); 619 goto out; 620 } 621 622 /* Configure the lo mode according to the input parameter */ 623 kni_net_config_lo_mode(lo_mode); 624 625 return 0; 626 627 out: 628 #ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS 629 unregister_pernet_subsys(&kni_net_ops); 630 #else 631 unregister_pernet_gen_subsys(kni_net_id, &kni_net_ops); 632 #endif 633 return rc; 634 } 635 636 static void __exit 637 kni_exit(void) 638 { 639 misc_deregister(&kni_misc); 640 #ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS 641 unregister_pernet_subsys(&kni_net_ops); 642 #else 643 unregister_pernet_gen_subsys(kni_net_id, &kni_net_ops); 644 #endif 645 } 646 647 module_init(kni_init); 648 module_exit(kni_exit); 649 650 module_param(lo_mode, charp, S_IRUGO | S_IWUSR); 651 MODULE_PARM_DESC(lo_mode, 652 "KNI loopback mode (default=lo_mode_none):\n" 653 " lo_mode_none Kernel loopback disabled\n" 654 " lo_mode_fifo Enable kernel loopback with fifo\n" 655 " lo_mode_fifo_skb Enable kernel loopback with fifo and skb buffer\n" 656 "\n" 657 ); 658 659 module_param(kthread_mode, charp, S_IRUGO); 660 MODULE_PARM_DESC(kthread_mode, 661 "Kernel thread mode (default=single):\n" 662 " single Single kernel thread mode enabled.\n" 663 " multiple Multiple kernel thread mode enabled.\n" 664 "\n" 665 ); 666