1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright(c) 2010-2014 Intel Corporation. 4 */ 5 6 #include <linux/version.h> 7 #include <linux/module.h> 8 #include <linux/miscdevice.h> 9 #include <linux/netdevice.h> 10 #include <linux/etherdevice.h> 11 #include <linux/pci.h> 12 #include <linux/kthread.h> 13 #include <linux/rwsem.h> 14 #include <linux/mutex.h> 15 #include <linux/nsproxy.h> 16 #include <net/net_namespace.h> 17 #include <net/netns/generic.h> 18 19 #include <rte_kni_common.h> 20 21 #include "compat.h" 22 #include "kni_dev.h" 23 24 MODULE_LICENSE("Dual BSD/GPL"); 25 MODULE_AUTHOR("Intel Corporation"); 26 MODULE_DESCRIPTION("Kernel Module for managing kni devices"); 27 28 #define KNI_RX_LOOP_NUM 1000 29 30 #define KNI_MAX_DEVICES 32 31 32 /* loopback mode */ 33 static char *lo_mode; 34 35 /* Kernel thread mode */ 36 static char *kthread_mode; 37 static uint32_t multiple_kthread_on; 38 39 /* Default carrier state for created KNI network interfaces */ 40 static char *carrier; 41 uint32_t dflt_carrier; 42 43 #define KNI_DEV_IN_USE_BIT_NUM 0 /* Bit number for device in use */ 44 45 static int kni_net_id; 46 47 struct kni_net { 48 unsigned long device_in_use; /* device in use flag */ 49 struct mutex kni_kthread_lock; 50 struct task_struct *kni_kthread; 51 struct rw_semaphore kni_list_lock; 52 struct list_head kni_list_head; 53 }; 54 55 static int __net_init 56 kni_init_net(struct net *net) 57 { 58 #ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS 59 struct kni_net *knet = net_generic(net, kni_net_id); 60 61 memset(knet, 0, sizeof(*knet)); 62 #else 63 struct kni_net *knet; 64 int ret; 65 66 knet = kzalloc(sizeof(struct kni_net), GFP_KERNEL); 67 if (!knet) { 68 ret = -ENOMEM; 69 return ret; 70 } 71 #endif 72 73 /* Clear the bit of device in use */ 74 clear_bit(KNI_DEV_IN_USE_BIT_NUM, &knet->device_in_use); 75 76 mutex_init(&knet->kni_kthread_lock); 77 78 init_rwsem(&knet->kni_list_lock); 79 INIT_LIST_HEAD(&knet->kni_list_head); 80 81 #ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS 82 return 0; 83 #else 84 ret = net_assign_generic(net, kni_net_id, knet); 85 if (ret < 0) 86 kfree(knet); 87 88 return ret; 89 #endif 90 } 91 92 static void __net_exit 93 kni_exit_net(struct net *net) 94 { 95 struct kni_net *knet __maybe_unused; 96 97 knet = net_generic(net, kni_net_id); 98 mutex_destroy(&knet->kni_kthread_lock); 99 100 #ifndef HAVE_SIMPLIFIED_PERNET_OPERATIONS 101 kfree(knet); 102 #endif 103 } 104 105 static struct pernet_operations kni_net_ops = { 106 .init = kni_init_net, 107 .exit = kni_exit_net, 108 #ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS 109 .id = &kni_net_id, 110 .size = sizeof(struct kni_net), 111 #endif 112 }; 113 114 static int 115 kni_thread_single(void *data) 116 { 117 struct kni_net *knet = data; 118 int j; 119 struct kni_dev *dev; 120 121 while (!kthread_should_stop()) { 122 down_read(&knet->kni_list_lock); 123 for (j = 0; j < KNI_RX_LOOP_NUM; j++) { 124 list_for_each_entry(dev, &knet->kni_list_head, list) { 125 kni_net_rx(dev); 126 kni_net_poll_resp(dev); 127 } 128 } 129 up_read(&knet->kni_list_lock); 130 #ifdef RTE_KNI_PREEMPT_DEFAULT 131 /* reschedule out for a while */ 132 schedule_timeout_interruptible( 133 usecs_to_jiffies(KNI_KTHREAD_RESCHEDULE_INTERVAL)); 134 #endif 135 } 136 137 return 0; 138 } 139 140 static int 141 kni_thread_multiple(void *param) 142 { 143 int j; 144 struct kni_dev *dev = param; 145 146 while (!kthread_should_stop()) { 147 for (j = 0; j < KNI_RX_LOOP_NUM; j++) { 148 kni_net_rx(dev); 149 kni_net_poll_resp(dev); 150 } 151 #ifdef RTE_KNI_PREEMPT_DEFAULT 152 schedule_timeout_interruptible( 153 usecs_to_jiffies(KNI_KTHREAD_RESCHEDULE_INTERVAL)); 154 #endif 155 } 156 157 return 0; 158 } 159 160 static int 161 kni_open(struct inode *inode, struct file *file) 162 { 163 struct net *net = current->nsproxy->net_ns; 164 struct kni_net *knet = net_generic(net, kni_net_id); 165 166 /* kni device can be opened by one user only per netns */ 167 if (test_and_set_bit(KNI_DEV_IN_USE_BIT_NUM, &knet->device_in_use)) 168 return -EBUSY; 169 170 file->private_data = get_net(net); 171 pr_debug("/dev/kni opened\n"); 172 173 return 0; 174 } 175 176 static int 177 kni_dev_remove(struct kni_dev *dev) 178 { 179 if (!dev) 180 return -ENODEV; 181 182 if (dev->net_dev) { 183 unregister_netdev(dev->net_dev); 184 free_netdev(dev->net_dev); 185 } 186 187 kni_net_release_fifo_phy(dev); 188 189 return 0; 190 } 191 192 static int 193 kni_release(struct inode *inode, struct file *file) 194 { 195 struct net *net = file->private_data; 196 struct kni_net *knet = net_generic(net, kni_net_id); 197 struct kni_dev *dev, *n; 198 199 /* Stop kernel thread for single mode */ 200 if (multiple_kthread_on == 0) { 201 mutex_lock(&knet->kni_kthread_lock); 202 /* Stop kernel thread */ 203 if (knet->kni_kthread != NULL) { 204 kthread_stop(knet->kni_kthread); 205 knet->kni_kthread = NULL; 206 } 207 mutex_unlock(&knet->kni_kthread_lock); 208 } 209 210 down_write(&knet->kni_list_lock); 211 list_for_each_entry_safe(dev, n, &knet->kni_list_head, list) { 212 /* Stop kernel thread for multiple mode */ 213 if (multiple_kthread_on && dev->pthread != NULL) { 214 kthread_stop(dev->pthread); 215 dev->pthread = NULL; 216 } 217 218 kni_dev_remove(dev); 219 list_del(&dev->list); 220 } 221 up_write(&knet->kni_list_lock); 222 223 /* Clear the bit of device in use */ 224 clear_bit(KNI_DEV_IN_USE_BIT_NUM, &knet->device_in_use); 225 226 put_net(net); 227 pr_debug("/dev/kni closed\n"); 228 229 return 0; 230 } 231 232 static int 233 kni_check_param(struct kni_dev *kni, struct rte_kni_device_info *dev) 234 { 235 if (!kni || !dev) 236 return -1; 237 238 /* Check if network name has been used */ 239 if (!strncmp(kni->name, dev->name, RTE_KNI_NAMESIZE)) { 240 pr_err("KNI name %s duplicated\n", dev->name); 241 return -1; 242 } 243 244 return 0; 245 } 246 247 static int 248 kni_run_thread(struct kni_net *knet, struct kni_dev *kni, uint8_t force_bind) 249 { 250 /** 251 * Create a new kernel thread for multiple mode, set its core affinity, 252 * and finally wake it up. 253 */ 254 if (multiple_kthread_on) { 255 kni->pthread = kthread_create(kni_thread_multiple, 256 (void *)kni, "kni_%s", kni->name); 257 if (IS_ERR(kni->pthread)) { 258 kni_dev_remove(kni); 259 return -ECANCELED; 260 } 261 262 if (force_bind) 263 kthread_bind(kni->pthread, kni->core_id); 264 wake_up_process(kni->pthread); 265 } else { 266 mutex_lock(&knet->kni_kthread_lock); 267 268 if (knet->kni_kthread == NULL) { 269 knet->kni_kthread = kthread_create(kni_thread_single, 270 (void *)knet, "kni_single"); 271 if (IS_ERR(knet->kni_kthread)) { 272 mutex_unlock(&knet->kni_kthread_lock); 273 kni_dev_remove(kni); 274 return -ECANCELED; 275 } 276 277 if (force_bind) 278 kthread_bind(knet->kni_kthread, kni->core_id); 279 wake_up_process(knet->kni_kthread); 280 } 281 282 mutex_unlock(&knet->kni_kthread_lock); 283 } 284 285 return 0; 286 } 287 288 static int 289 kni_ioctl_create(struct net *net, uint32_t ioctl_num, 290 unsigned long ioctl_param) 291 { 292 struct kni_net *knet = net_generic(net, kni_net_id); 293 int ret; 294 struct rte_kni_device_info dev_info; 295 struct net_device *net_dev = NULL; 296 struct kni_dev *kni, *dev, *n; 297 298 pr_info("Creating kni...\n"); 299 /* Check the buffer size, to avoid warning */ 300 if (_IOC_SIZE(ioctl_num) > sizeof(dev_info)) 301 return -EINVAL; 302 303 /* Copy kni info from user space */ 304 ret = copy_from_user(&dev_info, (void *)ioctl_param, sizeof(dev_info)); 305 if (ret) { 306 pr_err("copy_from_user in kni_ioctl_create"); 307 return -EIO; 308 } 309 310 /* Check if name is zero-ended */ 311 if (strnlen(dev_info.name, sizeof(dev_info.name)) == sizeof(dev_info.name)) { 312 pr_err("kni.name not zero-terminated"); 313 return -EINVAL; 314 } 315 316 /** 317 * Check if the cpu core id is valid for binding. 318 */ 319 if (dev_info.force_bind && !cpu_online(dev_info.core_id)) { 320 pr_err("cpu %u is not online\n", dev_info.core_id); 321 return -EINVAL; 322 } 323 324 /* Check if it has been created */ 325 down_read(&knet->kni_list_lock); 326 list_for_each_entry_safe(dev, n, &knet->kni_list_head, list) { 327 if (kni_check_param(dev, &dev_info) < 0) { 328 up_read(&knet->kni_list_lock); 329 return -EINVAL; 330 } 331 } 332 up_read(&knet->kni_list_lock); 333 334 net_dev = alloc_netdev(sizeof(struct kni_dev), dev_info.name, 335 #ifdef NET_NAME_USER 336 NET_NAME_USER, 337 #endif 338 kni_net_init); 339 if (net_dev == NULL) { 340 pr_err("error allocating device \"%s\"\n", dev_info.name); 341 return -EBUSY; 342 } 343 344 dev_net_set(net_dev, net); 345 346 kni = netdev_priv(net_dev); 347 348 kni->net_dev = net_dev; 349 kni->group_id = dev_info.group_id; 350 kni->core_id = dev_info.core_id; 351 strncpy(kni->name, dev_info.name, RTE_KNI_NAMESIZE); 352 353 /* Translate user space info into kernel space info */ 354 kni->tx_q = phys_to_virt(dev_info.tx_phys); 355 kni->rx_q = phys_to_virt(dev_info.rx_phys); 356 kni->alloc_q = phys_to_virt(dev_info.alloc_phys); 357 kni->free_q = phys_to_virt(dev_info.free_phys); 358 359 kni->req_q = phys_to_virt(dev_info.req_phys); 360 kni->resp_q = phys_to_virt(dev_info.resp_phys); 361 kni->sync_va = dev_info.sync_va; 362 kni->sync_kva = phys_to_virt(dev_info.sync_phys); 363 364 kni->mbuf_size = dev_info.mbuf_size; 365 366 pr_debug("tx_phys: 0x%016llx, tx_q addr: 0x%p\n", 367 (unsigned long long) dev_info.tx_phys, kni->tx_q); 368 pr_debug("rx_phys: 0x%016llx, rx_q addr: 0x%p\n", 369 (unsigned long long) dev_info.rx_phys, kni->rx_q); 370 pr_debug("alloc_phys: 0x%016llx, alloc_q addr: 0x%p\n", 371 (unsigned long long) dev_info.alloc_phys, kni->alloc_q); 372 pr_debug("free_phys: 0x%016llx, free_q addr: 0x%p\n", 373 (unsigned long long) dev_info.free_phys, kni->free_q); 374 pr_debug("req_phys: 0x%016llx, req_q addr: 0x%p\n", 375 (unsigned long long) dev_info.req_phys, kni->req_q); 376 pr_debug("resp_phys: 0x%016llx, resp_q addr: 0x%p\n", 377 (unsigned long long) dev_info.resp_phys, kni->resp_q); 378 pr_debug("mbuf_size: %u\n", kni->mbuf_size); 379 380 /* if user has provided a valid mac address */ 381 if (is_valid_ether_addr(dev_info.mac_addr)) 382 memcpy(net_dev->dev_addr, dev_info.mac_addr, ETH_ALEN); 383 else 384 /* 385 * Generate random mac address. eth_random_addr() is the 386 * newer version of generating mac address in kernel. 387 */ 388 random_ether_addr(net_dev->dev_addr); 389 390 if (dev_info.mtu) 391 net_dev->mtu = dev_info.mtu; 392 #ifdef HAVE_MAX_MTU_PARAM 393 net_dev->max_mtu = net_dev->mtu; 394 #endif 395 396 ret = register_netdev(net_dev); 397 if (ret) { 398 pr_err("error %i registering device \"%s\"\n", 399 ret, dev_info.name); 400 kni->net_dev = NULL; 401 kni_dev_remove(kni); 402 free_netdev(net_dev); 403 return -ENODEV; 404 } 405 406 netif_carrier_off(net_dev); 407 408 ret = kni_run_thread(knet, kni, dev_info.force_bind); 409 if (ret != 0) 410 return ret; 411 412 down_write(&knet->kni_list_lock); 413 list_add(&kni->list, &knet->kni_list_head); 414 up_write(&knet->kni_list_lock); 415 416 return 0; 417 } 418 419 static int 420 kni_ioctl_release(struct net *net, uint32_t ioctl_num, 421 unsigned long ioctl_param) 422 { 423 struct kni_net *knet = net_generic(net, kni_net_id); 424 int ret = -EINVAL; 425 struct kni_dev *dev, *n; 426 struct rte_kni_device_info dev_info; 427 428 if (_IOC_SIZE(ioctl_num) > sizeof(dev_info)) 429 return -EINVAL; 430 431 ret = copy_from_user(&dev_info, (void *)ioctl_param, sizeof(dev_info)); 432 if (ret) { 433 pr_err("copy_from_user in kni_ioctl_release"); 434 return -EIO; 435 } 436 437 /* Release the network device according to its name */ 438 if (strlen(dev_info.name) == 0) 439 return ret; 440 441 down_write(&knet->kni_list_lock); 442 list_for_each_entry_safe(dev, n, &knet->kni_list_head, list) { 443 if (strncmp(dev->name, dev_info.name, RTE_KNI_NAMESIZE) != 0) 444 continue; 445 446 if (multiple_kthread_on && dev->pthread != NULL) { 447 kthread_stop(dev->pthread); 448 dev->pthread = NULL; 449 } 450 451 kni_dev_remove(dev); 452 list_del(&dev->list); 453 ret = 0; 454 break; 455 } 456 up_write(&knet->kni_list_lock); 457 pr_info("%s release kni named %s\n", 458 (ret == 0 ? "Successfully" : "Unsuccessfully"), dev_info.name); 459 460 return ret; 461 } 462 463 static int 464 kni_ioctl(struct inode *inode, uint32_t ioctl_num, unsigned long ioctl_param) 465 { 466 int ret = -EINVAL; 467 struct net *net = current->nsproxy->net_ns; 468 469 pr_debug("IOCTL num=0x%0x param=0x%0lx\n", ioctl_num, ioctl_param); 470 471 /* 472 * Switch according to the ioctl called 473 */ 474 switch (_IOC_NR(ioctl_num)) { 475 case _IOC_NR(RTE_KNI_IOCTL_TEST): 476 /* For test only, not used */ 477 break; 478 case _IOC_NR(RTE_KNI_IOCTL_CREATE): 479 ret = kni_ioctl_create(net, ioctl_num, ioctl_param); 480 break; 481 case _IOC_NR(RTE_KNI_IOCTL_RELEASE): 482 ret = kni_ioctl_release(net, ioctl_num, ioctl_param); 483 break; 484 default: 485 pr_debug("IOCTL default\n"); 486 break; 487 } 488 489 return ret; 490 } 491 492 static int 493 kni_compat_ioctl(struct inode *inode, uint32_t ioctl_num, 494 unsigned long ioctl_param) 495 { 496 /* 32 bits app on 64 bits OS to be supported later */ 497 pr_debug("Not implemented.\n"); 498 499 return -EINVAL; 500 } 501 502 static const struct file_operations kni_fops = { 503 .owner = THIS_MODULE, 504 .open = kni_open, 505 .release = kni_release, 506 .unlocked_ioctl = (void *)kni_ioctl, 507 .compat_ioctl = (void *)kni_compat_ioctl, 508 }; 509 510 static struct miscdevice kni_misc = { 511 .minor = MISC_DYNAMIC_MINOR, 512 .name = KNI_DEVICE, 513 .fops = &kni_fops, 514 }; 515 516 static int __init 517 kni_parse_kthread_mode(void) 518 { 519 if (!kthread_mode) 520 return 0; 521 522 if (strcmp(kthread_mode, "single") == 0) 523 return 0; 524 else if (strcmp(kthread_mode, "multiple") == 0) 525 multiple_kthread_on = 1; 526 else 527 return -1; 528 529 return 0; 530 } 531 532 static int __init 533 kni_parse_carrier_state(void) 534 { 535 if (!carrier) { 536 dflt_carrier = 0; 537 return 0; 538 } 539 540 if (strcmp(carrier, "off") == 0) 541 dflt_carrier = 0; 542 else if (strcmp(carrier, "on") == 0) 543 dflt_carrier = 1; 544 else 545 return -1; 546 547 return 0; 548 } 549 550 static int __init 551 kni_init(void) 552 { 553 int rc; 554 555 if (kni_parse_kthread_mode() < 0) { 556 pr_err("Invalid parameter for kthread_mode\n"); 557 return -EINVAL; 558 } 559 560 if (multiple_kthread_on == 0) 561 pr_debug("Single kernel thread for all KNI devices\n"); 562 else 563 pr_debug("Multiple kernel thread mode enabled\n"); 564 565 if (kni_parse_carrier_state() < 0) { 566 pr_err("Invalid parameter for carrier\n"); 567 return -EINVAL; 568 } 569 570 if (dflt_carrier == 0) 571 pr_debug("Default carrier state set to off.\n"); 572 else 573 pr_debug("Default carrier state set to on.\n"); 574 575 #ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS 576 rc = register_pernet_subsys(&kni_net_ops); 577 #else 578 rc = register_pernet_gen_subsys(&kni_net_id, &kni_net_ops); 579 #endif 580 if (rc) 581 return -EPERM; 582 583 rc = misc_register(&kni_misc); 584 if (rc != 0) { 585 pr_err("Misc registration failed\n"); 586 goto out; 587 } 588 589 /* Configure the lo mode according to the input parameter */ 590 kni_net_config_lo_mode(lo_mode); 591 592 return 0; 593 594 out: 595 #ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS 596 unregister_pernet_subsys(&kni_net_ops); 597 #else 598 unregister_pernet_gen_subsys(kni_net_id, &kni_net_ops); 599 #endif 600 return rc; 601 } 602 603 static void __exit 604 kni_exit(void) 605 { 606 misc_deregister(&kni_misc); 607 #ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS 608 unregister_pernet_subsys(&kni_net_ops); 609 #else 610 unregister_pernet_gen_subsys(kni_net_id, &kni_net_ops); 611 #endif 612 } 613 614 module_init(kni_init); 615 module_exit(kni_exit); 616 617 module_param(lo_mode, charp, 0644); 618 MODULE_PARM_DESC(lo_mode, 619 "KNI loopback mode (default=lo_mode_none):\n" 620 "\t\tlo_mode_none Kernel loopback disabled\n" 621 "\t\tlo_mode_fifo Enable kernel loopback with fifo\n" 622 "\t\tlo_mode_fifo_skb Enable kernel loopback with fifo and skb buffer\n" 623 "\t\t" 624 ); 625 626 module_param(kthread_mode, charp, 0644); 627 MODULE_PARM_DESC(kthread_mode, 628 "Kernel thread mode (default=single):\n" 629 "\t\tsingle Single kernel thread mode enabled.\n" 630 "\t\tmultiple Multiple kernel thread mode enabled.\n" 631 "\t\t" 632 ); 633 634 module_param(carrier, charp, 0644); 635 MODULE_PARM_DESC(carrier, 636 "Default carrier state for KNI interface (default=off):\n" 637 "\t\toff Interfaces will be created with carrier state set to off.\n" 638 "\t\ton Interfaces will be created with carrier state set to on.\n" 639 "\t\t" 640 ); 641