1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2008-2017 Cisco Systems, Inc. All rights reserved. 3 * Copyright 2007 Nuova Systems, Inc. All rights reserved. 4 */ 5 6 #include <rte_memzone.h> 7 #include <rte_memcpy.h> 8 #include <rte_string_fns.h> 9 10 #include "vnic_dev.h" 11 #include "vnic_resource.h" 12 #include "vnic_devcmd.h" 13 #include "vnic_nic.h" 14 #include "vnic_stats.h" 15 16 17 enum vnic_proxy_type { 18 PROXY_NONE, 19 PROXY_BY_BDF, 20 PROXY_BY_INDEX, 21 }; 22 23 struct vnic_res { 24 void __iomem *vaddr; 25 dma_addr_t bus_addr; 26 unsigned int count; 27 }; 28 29 struct vnic_intr_coal_timer_info { 30 u32 mul; 31 u32 div; 32 u32 max_usec; 33 }; 34 35 struct vnic_dev { 36 void *priv; 37 struct rte_pci_device *pdev; 38 struct vnic_res res[RES_TYPE_MAX]; 39 enum vnic_dev_intr_mode intr_mode; 40 struct vnic_devcmd __iomem *devcmd; 41 struct vnic_devcmd_notify *notify; 42 struct vnic_devcmd_notify notify_copy; 43 dma_addr_t notify_pa; 44 u32 notify_sz; 45 dma_addr_t linkstatus_pa; 46 struct vnic_stats *stats; 47 dma_addr_t stats_pa; 48 struct vnic_devcmd_fw_info *fw_info; 49 dma_addr_t fw_info_pa; 50 enum vnic_proxy_type proxy; 51 u32 proxy_index; 52 u64 args[VNIC_DEVCMD_NARGS]; 53 int in_reset; 54 struct vnic_intr_coal_timer_info intr_coal_timer_info; 55 void *(*alloc_consistent)(void *priv, size_t size, 56 dma_addr_t *dma_handle, u8 *name); 57 void (*free_consistent)(void *priv, 58 size_t size, void *vaddr, 59 dma_addr_t dma_handle); 60 struct vnic_counter_counts *flow_counters; 61 dma_addr_t flow_counters_pa; 62 u8 flow_counters_dma_active; 63 }; 64 65 #define VNIC_MAX_RES_HDR_SIZE \ 66 (sizeof(struct vnic_resource_header) + \ 67 sizeof(struct vnic_resource) * RES_TYPE_MAX) 68 #define VNIC_RES_STRIDE 128 69 70 #define VNIC_MAX_FLOW_COUNTERS 2048 71 72 void *vnic_dev_priv(struct vnic_dev *vdev) 73 { 74 return vdev->priv; 75 } 76 77 void vnic_register_cbacks(struct vnic_dev *vdev, 78 void *(*alloc_consistent)(void *priv, size_t size, 79 dma_addr_t *dma_handle, u8 *name), 80 void (*free_consistent)(void *priv, 81 size_t size, void *vaddr, 82 dma_addr_t dma_handle)) 83 { 84 vdev->alloc_consistent = alloc_consistent; 85 vdev->free_consistent = free_consistent; 86 } 87 88 static int vnic_dev_discover_res(struct vnic_dev *vdev, 89 struct vnic_dev_bar *bar, unsigned int num_bars) 90 { 91 struct vnic_resource_header __iomem *rh; 92 struct mgmt_barmap_hdr __iomem *mrh; 93 struct vnic_resource __iomem *r; 94 u8 type; 95 96 if (num_bars == 0) 97 return -EINVAL; 98 99 if (bar->len < VNIC_MAX_RES_HDR_SIZE) { 100 pr_err("vNIC BAR0 res hdr length error\n"); 101 return -EINVAL; 102 } 103 104 rh = bar->vaddr; 105 mrh = bar->vaddr; 106 if (!rh) { 107 pr_err("vNIC BAR0 res hdr not mem-mapped\n"); 108 return -EINVAL; 109 } 110 111 /* Check for mgmt vnic in addition to normal vnic */ 112 if ((ioread32(&rh->magic) != VNIC_RES_MAGIC) || 113 (ioread32(&rh->version) != VNIC_RES_VERSION)) { 114 if ((ioread32(&mrh->magic) != MGMTVNIC_MAGIC) || 115 (ioread32(&mrh->version) != MGMTVNIC_VERSION)) { 116 pr_err("vNIC BAR0 res magic/version error " \ 117 "exp (%lx/%lx) or (%lx/%lx), curr (%x/%x)\n", 118 VNIC_RES_MAGIC, VNIC_RES_VERSION, 119 MGMTVNIC_MAGIC, MGMTVNIC_VERSION, 120 ioread32(&rh->magic), ioread32(&rh->version)); 121 return -EINVAL; 122 } 123 } 124 125 if (ioread32(&mrh->magic) == MGMTVNIC_MAGIC) 126 r = (struct vnic_resource __iomem *)(mrh + 1); 127 else 128 r = (struct vnic_resource __iomem *)(rh + 1); 129 130 131 while ((type = ioread8(&r->type)) != RES_TYPE_EOL) { 132 u8 bar_num = ioread8(&r->bar); 133 u32 bar_offset = ioread32(&r->bar_offset); 134 u32 count = ioread32(&r->count); 135 u32 len; 136 137 r++; 138 139 if (bar_num >= num_bars) 140 continue; 141 142 if (!bar[bar_num].len || !bar[bar_num].vaddr) 143 continue; 144 145 switch (type) { 146 case RES_TYPE_WQ: 147 case RES_TYPE_RQ: 148 case RES_TYPE_CQ: 149 case RES_TYPE_INTR_CTRL: 150 /* each count is stride bytes long */ 151 len = count * VNIC_RES_STRIDE; 152 if (len + bar_offset > bar[bar_num].len) { 153 pr_err("vNIC BAR0 resource %d " \ 154 "out-of-bounds, offset 0x%x + " \ 155 "size 0x%x > bar len 0x%lx\n", 156 type, bar_offset, 157 len, 158 bar[bar_num].len); 159 return -EINVAL; 160 } 161 break; 162 case RES_TYPE_INTR_PBA_LEGACY: 163 case RES_TYPE_DEVCMD: 164 len = count; 165 break; 166 default: 167 continue; 168 } 169 170 vdev->res[type].count = count; 171 vdev->res[type].vaddr = (char __iomem *)bar[bar_num].vaddr + 172 bar_offset; 173 vdev->res[type].bus_addr = bar[bar_num].bus_addr + bar_offset; 174 } 175 176 return 0; 177 } 178 179 unsigned int vnic_dev_get_res_count(struct vnic_dev *vdev, 180 enum vnic_res_type type) 181 { 182 return vdev->res[type].count; 183 } 184 185 void __iomem *vnic_dev_get_res(struct vnic_dev *vdev, enum vnic_res_type type, 186 unsigned int index) 187 { 188 if (!vdev->res[type].vaddr) 189 return NULL; 190 191 switch (type) { 192 case RES_TYPE_WQ: 193 case RES_TYPE_RQ: 194 case RES_TYPE_CQ: 195 case RES_TYPE_INTR_CTRL: 196 return (char __iomem *)vdev->res[type].vaddr + 197 index * VNIC_RES_STRIDE; 198 default: 199 return (char __iomem *)vdev->res[type].vaddr; 200 } 201 } 202 203 unsigned int vnic_dev_desc_ring_size(struct vnic_dev_ring *ring, 204 unsigned int desc_count, unsigned int desc_size) 205 { 206 /* The base address of the desc rings must be 512 byte aligned. 207 * Descriptor count is aligned to groups of 32 descriptors. A 208 * count of 0 means the maximum 4096 descriptors. Descriptor 209 * size is aligned to 16 bytes. 210 */ 211 212 unsigned int count_align = 32; 213 unsigned int desc_align = 16; 214 215 ring->base_align = 512; 216 217 if (desc_count == 0) 218 desc_count = 4096; 219 220 ring->desc_count = VNIC_ALIGN(desc_count, count_align); 221 222 ring->desc_size = VNIC_ALIGN(desc_size, desc_align); 223 224 ring->size = ring->desc_count * ring->desc_size; 225 ring->size_unaligned = ring->size + ring->base_align; 226 227 return ring->size_unaligned; 228 } 229 230 void vnic_dev_clear_desc_ring(struct vnic_dev_ring *ring) 231 { 232 memset(ring->descs, 0, ring->size); 233 } 234 235 int vnic_dev_alloc_desc_ring(struct vnic_dev *vdev, 236 struct vnic_dev_ring *ring, 237 unsigned int desc_count, unsigned int desc_size, 238 __attribute__((unused)) unsigned int socket_id, 239 char *z_name) 240 { 241 void *alloc_addr; 242 dma_addr_t alloc_pa = 0; 243 244 vnic_dev_desc_ring_size(ring, desc_count, desc_size); 245 alloc_addr = vdev->alloc_consistent(vdev->priv, 246 ring->size_unaligned, 247 &alloc_pa, (u8 *)z_name); 248 if (!alloc_addr) { 249 pr_err("Failed to allocate ring (size=%d), aborting\n", 250 (int)ring->size); 251 return -ENOMEM; 252 } 253 ring->descs_unaligned = alloc_addr; 254 if (!alloc_pa) { 255 pr_err("Failed to map allocated ring (size=%d), aborting\n", 256 (int)ring->size); 257 vdev->free_consistent(vdev->priv, 258 ring->size_unaligned, 259 alloc_addr, 260 alloc_pa); 261 return -ENOMEM; 262 } 263 ring->base_addr_unaligned = alloc_pa; 264 265 ring->base_addr = VNIC_ALIGN(ring->base_addr_unaligned, 266 ring->base_align); 267 ring->descs = (u8 *)ring->descs_unaligned + 268 (ring->base_addr - ring->base_addr_unaligned); 269 270 vnic_dev_clear_desc_ring(ring); 271 272 ring->desc_avail = ring->desc_count - 1; 273 274 return 0; 275 } 276 277 void vnic_dev_free_desc_ring(__attribute__((unused)) struct vnic_dev *vdev, 278 struct vnic_dev_ring *ring) 279 { 280 if (ring->descs) { 281 vdev->free_consistent(vdev->priv, 282 ring->size_unaligned, 283 ring->descs_unaligned, 284 ring->base_addr_unaligned); 285 ring->descs = NULL; 286 } 287 } 288 289 static int _vnic_dev_cmd(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd, 290 int wait) 291 { 292 struct vnic_devcmd __iomem *devcmd = vdev->devcmd; 293 unsigned int i; 294 int delay; 295 u32 status; 296 int err; 297 298 status = ioread32(&devcmd->status); 299 if (status == 0xFFFFFFFF) { 300 /* PCI-e target device is gone */ 301 return -ENODEV; 302 } 303 if (status & STAT_BUSY) { 304 305 pr_err("Busy devcmd %d\n", _CMD_N(cmd)); 306 return -EBUSY; 307 } 308 309 if (_CMD_DIR(cmd) & _CMD_DIR_WRITE) { 310 for (i = 0; i < VNIC_DEVCMD_NARGS; i++) 311 writeq(vdev->args[i], &devcmd->args[i]); 312 wmb(); /* complete all writes initiated till now */ 313 } 314 315 iowrite32(cmd, &devcmd->cmd); 316 317 if ((_CMD_FLAGS(cmd) & _CMD_FLAGS_NOWAIT)) 318 return 0; 319 320 for (delay = 0; delay < wait; delay++) { 321 322 udelay(100); 323 324 status = ioread32(&devcmd->status); 325 if (status == 0xFFFFFFFF) { 326 /* PCI-e target device is gone */ 327 return -ENODEV; 328 } 329 330 if (!(status & STAT_BUSY)) { 331 if (status & STAT_ERROR) { 332 err = -(int)readq(&devcmd->args[0]); 333 if (cmd != CMD_CAPABILITY) 334 pr_err("Devcmd %d failed " \ 335 "with error code %d\n", 336 _CMD_N(cmd), err); 337 return err; 338 } 339 340 if (_CMD_DIR(cmd) & _CMD_DIR_READ) { 341 rmb();/* finish all reads initiated till now */ 342 for (i = 0; i < VNIC_DEVCMD_NARGS; i++) 343 vdev->args[i] = readq(&devcmd->args[i]); 344 } 345 346 return 0; 347 } 348 } 349 350 pr_err("Timedout devcmd %d\n", _CMD_N(cmd)); 351 return -ETIMEDOUT; 352 } 353 354 static int vnic_dev_cmd_proxy(struct vnic_dev *vdev, 355 enum vnic_devcmd_cmd proxy_cmd, enum vnic_devcmd_cmd cmd, 356 u64 *args, int nargs, int wait) 357 { 358 u32 status; 359 int err; 360 361 /* 362 * Proxy command consumes 2 arguments. One for proxy index, 363 * the other is for command to be proxied 364 */ 365 if (nargs > VNIC_DEVCMD_NARGS - 2) { 366 pr_err("number of args %d exceeds the maximum\n", nargs); 367 return -EINVAL; 368 } 369 memset(vdev->args, 0, sizeof(vdev->args)); 370 371 vdev->args[0] = vdev->proxy_index; 372 vdev->args[1] = cmd; 373 memcpy(&vdev->args[2], args, nargs * sizeof(args[0])); 374 375 err = _vnic_dev_cmd(vdev, proxy_cmd, wait); 376 if (err) 377 return err; 378 379 status = (u32)vdev->args[0]; 380 if (status & STAT_ERROR) { 381 err = (int)vdev->args[1]; 382 if (err != ERR_ECMDUNKNOWN || 383 cmd != CMD_CAPABILITY) 384 pr_err("Error %d proxy devcmd %d\n", err, _CMD_N(cmd)); 385 return err; 386 } 387 388 memcpy(args, &vdev->args[1], nargs * sizeof(args[0])); 389 390 return 0; 391 } 392 393 static int vnic_dev_cmd_no_proxy(struct vnic_dev *vdev, 394 enum vnic_devcmd_cmd cmd, u64 *args, int nargs, int wait) 395 { 396 int err; 397 398 if (nargs > VNIC_DEVCMD_NARGS) { 399 pr_err("number of args %d exceeds the maximum\n", nargs); 400 return -EINVAL; 401 } 402 memset(vdev->args, 0, sizeof(vdev->args)); 403 memcpy(vdev->args, args, nargs * sizeof(args[0])); 404 405 err = _vnic_dev_cmd(vdev, cmd, wait); 406 407 memcpy(args, vdev->args, nargs * sizeof(args[0])); 408 409 return err; 410 } 411 412 int vnic_dev_cmd(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd, 413 u64 *a0, u64 *a1, int wait) 414 { 415 u64 args[2]; 416 int err; 417 418 args[0] = *a0; 419 args[1] = *a1; 420 memset(vdev->args, 0, sizeof(vdev->args)); 421 422 switch (vdev->proxy) { 423 case PROXY_BY_INDEX: 424 err = vnic_dev_cmd_proxy(vdev, CMD_PROXY_BY_INDEX, cmd, 425 args, ARRAY_SIZE(args), wait); 426 break; 427 case PROXY_BY_BDF: 428 err = vnic_dev_cmd_proxy(vdev, CMD_PROXY_BY_BDF, cmd, 429 args, ARRAY_SIZE(args), wait); 430 break; 431 case PROXY_NONE: 432 default: 433 err = vnic_dev_cmd_no_proxy(vdev, cmd, args, 2, wait); 434 break; 435 } 436 437 if (err == 0) { 438 *a0 = args[0]; 439 *a1 = args[1]; 440 } 441 442 return err; 443 } 444 445 int vnic_dev_cmd_args(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd, 446 u64 *args, int nargs, int wait) 447 { 448 switch (vdev->proxy) { 449 case PROXY_BY_INDEX: 450 return vnic_dev_cmd_proxy(vdev, CMD_PROXY_BY_INDEX, cmd, 451 args, nargs, wait); 452 case PROXY_BY_BDF: 453 return vnic_dev_cmd_proxy(vdev, CMD_PROXY_BY_BDF, cmd, 454 args, nargs, wait); 455 case PROXY_NONE: 456 default: 457 return vnic_dev_cmd_no_proxy(vdev, cmd, args, nargs, wait); 458 } 459 } 460 461 int vnic_dev_fw_info(struct vnic_dev *vdev, 462 struct vnic_devcmd_fw_info **fw_info) 463 { 464 char name[NAME_MAX]; 465 u64 a0, a1 = 0; 466 int wait = 1000; 467 int err = 0; 468 static u32 instance; 469 470 if (!vdev->fw_info) { 471 snprintf((char *)name, sizeof(name), "vnic_fw_info-%u", 472 instance++); 473 vdev->fw_info = vdev->alloc_consistent(vdev->priv, 474 sizeof(struct vnic_devcmd_fw_info), 475 &vdev->fw_info_pa, (u8 *)name); 476 if (!vdev->fw_info) 477 return -ENOMEM; 478 a0 = vdev->fw_info_pa; 479 a1 = sizeof(struct vnic_devcmd_fw_info); 480 err = vnic_dev_cmd(vdev, CMD_MCPU_FW_INFO, 481 &a0, &a1, wait); 482 } 483 *fw_info = vdev->fw_info; 484 return err; 485 } 486 487 static int vnic_dev_advanced_filters_cap(struct vnic_dev *vdev, u64 *args, 488 int nargs) 489 { 490 memset(args, 0, nargs * sizeof(*args)); 491 args[0] = CMD_ADD_ADV_FILTER; 492 args[1] = FILTER_CAP_MODE_V1_FLAG; 493 return vnic_dev_cmd_args(vdev, CMD_CAPABILITY, args, nargs, 1000); 494 } 495 496 int vnic_dev_capable_adv_filters(struct vnic_dev *vdev) 497 { 498 u64 a0 = CMD_ADD_ADV_FILTER, a1 = 0; 499 int wait = 1000; 500 int err; 501 502 err = vnic_dev_cmd(vdev, CMD_CAPABILITY, &a0, &a1, wait); 503 if (err) 504 return 0; 505 return (a1 >= (u32)FILTER_DPDK_1); 506 } 507 508 /* Determine the "best" filtering mode VIC is capaible of. Returns one of 3 509 * value or 0 on error: 510 * FILTER_DPDK_1- advanced filters availabile 511 * FILTER_USNIC_IP_FLAG - advanced filters but with the restriction that 512 * the IP layer must explicitly specified. I.e. cannot have a UDP 513 * filter that matches both IPv4 and IPv6. 514 * FILTER_IPV4_5TUPLE - fallback if either of the 2 above aren't available. 515 * all other filter types are not available. 516 * Retrun true in filter_tags if supported 517 */ 518 int vnic_dev_capable_filter_mode(struct vnic_dev *vdev, u32 *mode, 519 u8 *filter_actions) 520 { 521 u64 args[4]; 522 int err; 523 u32 max_level = 0; 524 525 err = vnic_dev_advanced_filters_cap(vdev, args, 4); 526 527 /* determine supported filter actions */ 528 *filter_actions = FILTER_ACTION_RQ_STEERING_FLAG; /* always available */ 529 if (args[2] == FILTER_CAP_MODE_V1) 530 *filter_actions = args[3]; 531 532 if (err || ((args[0] == 1) && (args[1] == 0))) { 533 /* Adv filter Command not supported or adv filters available but 534 * not enabled. Try the normal filter capability command. 535 */ 536 args[0] = CMD_ADD_FILTER; 537 args[1] = 0; 538 err = vnic_dev_cmd_args(vdev, CMD_CAPABILITY, args, 2, 1000); 539 if (err) 540 return err; 541 max_level = args[1]; 542 goto parse_max_level; 543 } else if (args[2] == FILTER_CAP_MODE_V1) { 544 /* parse filter capability mask in args[1] */ 545 if (args[1] & FILTER_DPDK_1_FLAG) 546 *mode = FILTER_DPDK_1; 547 else if (args[1] & FILTER_USNIC_IP_FLAG) 548 *mode = FILTER_USNIC_IP; 549 else if (args[1] & FILTER_IPV4_5TUPLE_FLAG) 550 *mode = FILTER_IPV4_5TUPLE; 551 return 0; 552 } 553 max_level = args[1]; 554 parse_max_level: 555 if (max_level >= (u32)FILTER_USNIC_IP) 556 *mode = FILTER_USNIC_IP; 557 else 558 *mode = FILTER_IPV4_5TUPLE; 559 return 0; 560 } 561 562 void vnic_dev_capable_udp_rss_weak(struct vnic_dev *vdev, bool *cfg_chk, 563 bool *weak) 564 { 565 u64 a0 = CMD_NIC_CFG, a1 = 0; 566 int wait = 1000; 567 int err; 568 569 *cfg_chk = false; 570 *weak = false; 571 err = vnic_dev_cmd(vdev, CMD_CAPABILITY, &a0, &a1, wait); 572 if (err == 0 && a0 != 0 && a1 != 0) { 573 *cfg_chk = true; 574 *weak = !!((a1 >> 32) & CMD_NIC_CFG_CAPF_UDP_WEAK); 575 } 576 } 577 578 int vnic_dev_capable(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd) 579 { 580 u64 a0 = (u32)cmd, a1 = 0; 581 int wait = 1000; 582 int err; 583 584 err = vnic_dev_cmd(vdev, CMD_CAPABILITY, &a0, &a1, wait); 585 586 return !(err || a0); 587 } 588 589 int vnic_dev_spec(struct vnic_dev *vdev, unsigned int offset, size_t size, 590 void *value) 591 { 592 u64 a0, a1; 593 int wait = 1000; 594 int err; 595 596 a0 = offset; 597 a1 = size; 598 599 err = vnic_dev_cmd(vdev, CMD_DEV_SPEC, &a0, &a1, wait); 600 601 switch (size) { 602 case 1: 603 *(u8 *)value = (u8)a0; 604 break; 605 case 2: 606 *(u16 *)value = (u16)a0; 607 break; 608 case 4: 609 *(u32 *)value = (u32)a0; 610 break; 611 case 8: 612 *(u64 *)value = a0; 613 break; 614 default: 615 BUG(); 616 break; 617 } 618 619 return err; 620 } 621 622 int vnic_dev_stats_clear(struct vnic_dev *vdev) 623 { 624 u64 a0 = 0, a1 = 0; 625 int wait = 1000; 626 627 return vnic_dev_cmd(vdev, CMD_STATS_CLEAR, &a0, &a1, wait); 628 } 629 630 int vnic_dev_stats_dump(struct vnic_dev *vdev, struct vnic_stats **stats) 631 { 632 u64 a0, a1; 633 int wait = 1000; 634 635 if (!vdev->stats) 636 return -ENOMEM; 637 638 *stats = vdev->stats; 639 a0 = vdev->stats_pa; 640 a1 = sizeof(struct vnic_stats); 641 642 return vnic_dev_cmd(vdev, CMD_STATS_DUMP, &a0, &a1, wait); 643 } 644 645 /* 646 * Configure counter DMA 647 */ 648 int vnic_dev_counter_dma_cfg(struct vnic_dev *vdev, u32 period, 649 u32 num_counters) 650 { 651 u64 args[3]; 652 int wait = 1000; 653 int err; 654 655 if (num_counters > VNIC_MAX_FLOW_COUNTERS) 656 return -ENOMEM; 657 if (period > 0 && (period < VNIC_COUNTER_DMA_MIN_PERIOD || 658 num_counters == 0)) 659 return -EINVAL; 660 661 args[0] = num_counters; 662 args[1] = vdev->flow_counters_pa; 663 args[2] = period; 664 err = vnic_dev_cmd_args(vdev, CMD_COUNTER_DMA_CONFIG, args, 3, wait); 665 666 /* record if DMAs need to be stopped on close */ 667 if (!err) 668 vdev->flow_counters_dma_active = (num_counters != 0 && 669 period != 0); 670 671 return err; 672 } 673 674 int vnic_dev_close(struct vnic_dev *vdev) 675 { 676 u64 a0 = 0, a1 = 0; 677 int wait = 1000; 678 679 return vnic_dev_cmd(vdev, CMD_CLOSE, &a0, &a1, wait); 680 } 681 682 int vnic_dev_enable_wait(struct vnic_dev *vdev) 683 { 684 u64 a0 = 0, a1 = 0; 685 int wait = 1000; 686 687 if (vnic_dev_capable(vdev, CMD_ENABLE_WAIT)) 688 return vnic_dev_cmd(vdev, CMD_ENABLE_WAIT, &a0, &a1, wait); 689 else 690 return vnic_dev_cmd(vdev, CMD_ENABLE, &a0, &a1, wait); 691 } 692 693 int vnic_dev_disable(struct vnic_dev *vdev) 694 { 695 u64 a0 = 0, a1 = 0; 696 int wait = 1000; 697 698 return vnic_dev_cmd(vdev, CMD_DISABLE, &a0, &a1, wait); 699 } 700 701 int vnic_dev_open(struct vnic_dev *vdev, int arg) 702 { 703 u64 a0 = (u32)arg, a1 = 0; 704 int wait = 1000; 705 706 return vnic_dev_cmd(vdev, CMD_OPEN, &a0, &a1, wait); 707 } 708 709 int vnic_dev_open_done(struct vnic_dev *vdev, int *done) 710 { 711 u64 a0 = 0, a1 = 0; 712 int wait = 1000; 713 int err; 714 715 *done = 0; 716 717 err = vnic_dev_cmd(vdev, CMD_OPEN_STATUS, &a0, &a1, wait); 718 if (err) 719 return err; 720 721 *done = (a0 == 0); 722 723 return 0; 724 } 725 726 int vnic_dev_get_mac_addr(struct vnic_dev *vdev, u8 *mac_addr) 727 { 728 u64 a0 = 0, a1 = 0; 729 int wait = 1000; 730 int err, i; 731 732 for (i = 0; i < ETH_ALEN; i++) 733 mac_addr[i] = 0; 734 735 err = vnic_dev_cmd(vdev, CMD_GET_MAC_ADDR, &a0, &a1, wait); 736 if (err) 737 return err; 738 739 for (i = 0; i < ETH_ALEN; i++) 740 mac_addr[i] = ((u8 *)&a0)[i]; 741 742 return 0; 743 } 744 745 int vnic_dev_packet_filter(struct vnic_dev *vdev, int directed, int multicast, 746 int broadcast, int promisc, int allmulti) 747 { 748 u64 a0, a1 = 0; 749 int wait = 1000; 750 int err; 751 752 a0 = (directed ? CMD_PFILTER_DIRECTED : 0) | 753 (multicast ? CMD_PFILTER_MULTICAST : 0) | 754 (broadcast ? CMD_PFILTER_BROADCAST : 0) | 755 (promisc ? CMD_PFILTER_PROMISCUOUS : 0) | 756 (allmulti ? CMD_PFILTER_ALL_MULTICAST : 0); 757 758 err = vnic_dev_cmd(vdev, CMD_PACKET_FILTER, &a0, &a1, wait); 759 if (err) 760 pr_err("Can't set packet filter\n"); 761 762 return err; 763 } 764 765 int vnic_dev_add_addr(struct vnic_dev *vdev, u8 *addr) 766 { 767 u64 a0 = 0, a1 = 0; 768 int wait = 1000; 769 int err; 770 int i; 771 772 for (i = 0; i < ETH_ALEN; i++) 773 ((u8 *)&a0)[i] = addr[i]; 774 775 err = vnic_dev_cmd(vdev, CMD_ADDR_ADD, &a0, &a1, wait); 776 if (err) 777 pr_err("Can't add addr [%02x:%02x:%02x:%02x:%02x:%02x], %d\n", 778 addr[0], addr[1], addr[2], addr[3], addr[4], addr[5], 779 err); 780 781 return err; 782 } 783 784 int vnic_dev_del_addr(struct vnic_dev *vdev, u8 *addr) 785 { 786 u64 a0 = 0, a1 = 0; 787 int wait = 1000; 788 int err; 789 int i; 790 791 for (i = 0; i < ETH_ALEN; i++) 792 ((u8 *)&a0)[i] = addr[i]; 793 794 err = vnic_dev_cmd(vdev, CMD_ADDR_DEL, &a0, &a1, wait); 795 if (err) 796 pr_err("Can't del addr [%02x:%02x:%02x:%02x:%02x:%02x], %d\n", 797 addr[0], addr[1], addr[2], addr[3], addr[4], addr[5], 798 err); 799 800 return err; 801 } 802 803 int vnic_dev_set_ig_vlan_rewrite_mode(struct vnic_dev *vdev, 804 u8 ig_vlan_rewrite_mode) 805 { 806 u64 a0 = ig_vlan_rewrite_mode, a1 = 0; 807 int wait = 1000; 808 809 if (vnic_dev_capable(vdev, CMD_IG_VLAN_REWRITE_MODE)) 810 return vnic_dev_cmd(vdev, CMD_IG_VLAN_REWRITE_MODE, 811 &a0, &a1, wait); 812 else 813 return 0; 814 } 815 816 void vnic_dev_set_reset_flag(struct vnic_dev *vdev, int state) 817 { 818 vdev->in_reset = state; 819 } 820 821 static inline int vnic_dev_in_reset(struct vnic_dev *vdev) 822 { 823 return vdev->in_reset; 824 } 825 826 int vnic_dev_notify_setcmd(struct vnic_dev *vdev, 827 void *notify_addr, dma_addr_t notify_pa, u16 intr) 828 { 829 u64 a0, a1; 830 int wait = 1000; 831 int r; 832 833 memset(notify_addr, 0, sizeof(struct vnic_devcmd_notify)); 834 if (!vnic_dev_in_reset(vdev)) { 835 vdev->notify = notify_addr; 836 vdev->notify_pa = notify_pa; 837 } 838 839 a0 = (u64)notify_pa; 840 a1 = ((u64)intr << 32) & 0x0000ffff00000000ULL; 841 a1 += sizeof(struct vnic_devcmd_notify); 842 843 r = vnic_dev_cmd(vdev, CMD_NOTIFY, &a0, &a1, wait); 844 if (!vnic_dev_in_reset(vdev)) 845 vdev->notify_sz = (r == 0) ? (u32)a1 : 0; 846 847 return r; 848 } 849 850 int vnic_dev_notify_set(struct vnic_dev *vdev, u16 intr) 851 { 852 void *notify_addr = NULL; 853 dma_addr_t notify_pa = 0; 854 char name[NAME_MAX]; 855 static u32 instance; 856 857 if (vdev->notify || vdev->notify_pa) { 858 return vnic_dev_notify_setcmd(vdev, vdev->notify, 859 vdev->notify_pa, intr); 860 } 861 if (!vnic_dev_in_reset(vdev)) { 862 snprintf((char *)name, sizeof(name), 863 "vnic_notify-%u", instance++); 864 notify_addr = vdev->alloc_consistent(vdev->priv, 865 sizeof(struct vnic_devcmd_notify), 866 ¬ify_pa, (u8 *)name); 867 if (!notify_addr) 868 return -ENOMEM; 869 } 870 871 return vnic_dev_notify_setcmd(vdev, notify_addr, notify_pa, intr); 872 } 873 874 int vnic_dev_notify_unsetcmd(struct vnic_dev *vdev) 875 { 876 u64 a0, a1; 877 int wait = 1000; 878 int err; 879 880 a0 = 0; /* paddr = 0 to unset notify buffer */ 881 a1 = 0x0000ffff00000000ULL; /* intr num = -1 to unreg for intr */ 882 a1 += sizeof(struct vnic_devcmd_notify); 883 884 err = vnic_dev_cmd(vdev, CMD_NOTIFY, &a0, &a1, wait); 885 if (!vnic_dev_in_reset(vdev)) { 886 vdev->notify = NULL; 887 vdev->notify_pa = 0; 888 vdev->notify_sz = 0; 889 } 890 891 return err; 892 } 893 894 int vnic_dev_notify_unset(struct vnic_dev *vdev) 895 { 896 if (vdev->notify && !vnic_dev_in_reset(vdev)) { 897 vdev->free_consistent(vdev->priv, 898 sizeof(struct vnic_devcmd_notify), 899 vdev->notify, 900 vdev->notify_pa); 901 } 902 903 return vnic_dev_notify_unsetcmd(vdev); 904 } 905 906 static int vnic_dev_notify_ready(struct vnic_dev *vdev) 907 { 908 u32 *words; 909 unsigned int nwords = vdev->notify_sz / 4; 910 unsigned int i; 911 u32 csum; 912 913 if (!vdev->notify || !vdev->notify_sz) 914 return 0; 915 916 do { 917 csum = 0; 918 rte_memcpy(&vdev->notify_copy, vdev->notify, vdev->notify_sz); 919 words = (u32 *)&vdev->notify_copy; 920 for (i = 1; i < nwords; i++) 921 csum += words[i]; 922 } while (csum != words[0]); 923 924 return 1; 925 } 926 927 int vnic_dev_init(struct vnic_dev *vdev, int arg) 928 { 929 u64 a0 = (u32)arg, a1 = 0; 930 int wait = 1000; 931 int r = 0; 932 933 if (vnic_dev_capable(vdev, CMD_INIT)) 934 r = vnic_dev_cmd(vdev, CMD_INIT, &a0, &a1, wait); 935 else { 936 vnic_dev_cmd(vdev, CMD_INIT_v1, &a0, &a1, wait); 937 if (a0 & CMD_INITF_DEFAULT_MAC) { 938 /* Emulate these for old CMD_INIT_v1 which 939 * didn't pass a0 so no CMD_INITF_*. 940 */ 941 vnic_dev_cmd(vdev, CMD_GET_MAC_ADDR, &a0, &a1, wait); 942 vnic_dev_cmd(vdev, CMD_ADDR_ADD, &a0, &a1, wait); 943 } 944 } 945 return r; 946 } 947 948 void vnic_dev_intr_coal_timer_info_default(struct vnic_dev *vdev) 949 { 950 /* Default: hardware intr coal timer is in units of 1.5 usecs */ 951 vdev->intr_coal_timer_info.mul = 2; 952 vdev->intr_coal_timer_info.div = 3; 953 vdev->intr_coal_timer_info.max_usec = 954 vnic_dev_intr_coal_timer_hw_to_usec(vdev, 0xffff); 955 } 956 957 int vnic_dev_link_status(struct vnic_dev *vdev) 958 { 959 if (!vnic_dev_notify_ready(vdev)) 960 return 0; 961 962 return vdev->notify_copy.link_state; 963 } 964 965 u32 vnic_dev_port_speed(struct vnic_dev *vdev) 966 { 967 if (!vnic_dev_notify_ready(vdev)) 968 return 0; 969 970 return vdev->notify_copy.port_speed; 971 } 972 973 u32 vnic_dev_intr_coal_timer_usec_to_hw(struct vnic_dev *vdev, u32 usec) 974 { 975 return (usec * vdev->intr_coal_timer_info.mul) / 976 vdev->intr_coal_timer_info.div; 977 } 978 979 u32 vnic_dev_intr_coal_timer_hw_to_usec(struct vnic_dev *vdev, u32 hw_cycles) 980 { 981 return (hw_cycles * vdev->intr_coal_timer_info.div) / 982 vdev->intr_coal_timer_info.mul; 983 } 984 985 u32 vnic_dev_get_intr_coal_timer_max(struct vnic_dev *vdev) 986 { 987 return vdev->intr_coal_timer_info.max_usec; 988 } 989 990 int vnic_dev_alloc_stats_mem(struct vnic_dev *vdev) 991 { 992 char name[NAME_MAX]; 993 static u32 instance; 994 995 snprintf((char *)name, sizeof(name), "vnic_stats-%u", instance++); 996 vdev->stats = vdev->alloc_consistent(vdev->priv, 997 sizeof(struct vnic_stats), 998 &vdev->stats_pa, (u8 *)name); 999 return vdev->stats == NULL ? -ENOMEM : 0; 1000 } 1001 1002 /* 1003 * Initialize for up to VNIC_MAX_FLOW_COUNTERS 1004 */ 1005 int vnic_dev_alloc_counter_mem(struct vnic_dev *vdev) 1006 { 1007 char name[NAME_MAX]; 1008 static u32 instance; 1009 1010 snprintf((char *)name, sizeof(name), "vnic_flow_ctrs-%u", instance++); 1011 vdev->flow_counters = vdev->alloc_consistent(vdev->priv, 1012 sizeof(struct vnic_counter_counts) 1013 * VNIC_MAX_FLOW_COUNTERS, 1014 &vdev->flow_counters_pa, 1015 (u8 *)name); 1016 vdev->flow_counters_dma_active = 0; 1017 return vdev->flow_counters == NULL ? -ENOMEM : 0; 1018 } 1019 1020 void vnic_dev_unregister(struct vnic_dev *vdev) 1021 { 1022 if (vdev) { 1023 if (vdev->notify) 1024 vdev->free_consistent(vdev->priv, 1025 sizeof(struct vnic_devcmd_notify), 1026 vdev->notify, 1027 vdev->notify_pa); 1028 if (vdev->stats) 1029 vdev->free_consistent(vdev->priv, 1030 sizeof(struct vnic_stats), 1031 vdev->stats, vdev->stats_pa); 1032 if (vdev->flow_counters) { 1033 /* turn off counter DMAs before freeing memory */ 1034 if (vdev->flow_counters_dma_active) 1035 vnic_dev_counter_dma_cfg(vdev, 0, 0); 1036 1037 vdev->free_consistent(vdev->priv, 1038 sizeof(struct vnic_counter_counts) 1039 * VNIC_MAX_FLOW_COUNTERS, 1040 vdev->flow_counters, vdev->flow_counters_pa); 1041 } 1042 if (vdev->fw_info) 1043 vdev->free_consistent(vdev->priv, 1044 sizeof(struct vnic_devcmd_fw_info), 1045 vdev->fw_info, vdev->fw_info_pa); 1046 rte_free(vdev); 1047 } 1048 } 1049 1050 struct vnic_dev *vnic_dev_register(struct vnic_dev *vdev, 1051 void *priv, struct rte_pci_device *pdev, struct vnic_dev_bar *bar, 1052 unsigned int num_bars) 1053 { 1054 if (!vdev) { 1055 char name[NAME_MAX]; 1056 snprintf((char *)name, sizeof(name), "%s-vnic", 1057 pdev->device.name); 1058 vdev = (struct vnic_dev *)rte_zmalloc_socket(name, 1059 sizeof(struct vnic_dev), 1060 RTE_CACHE_LINE_SIZE, 1061 pdev->device.numa_node); 1062 if (!vdev) 1063 return NULL; 1064 } 1065 1066 vdev->priv = priv; 1067 vdev->pdev = pdev; 1068 1069 if (vnic_dev_discover_res(vdev, bar, num_bars)) 1070 goto err_out; 1071 1072 vdev->devcmd = vnic_dev_get_res(vdev, RES_TYPE_DEVCMD, 0); 1073 if (!vdev->devcmd) 1074 goto err_out; 1075 1076 return vdev; 1077 1078 err_out: 1079 vnic_dev_unregister(vdev); 1080 return NULL; 1081 } 1082 1083 /* 1084 * vnic_dev_classifier: Add/Delete classifier entries 1085 * @vdev: vdev of the device 1086 * @cmd: CLSF_ADD for Add filter 1087 * CLSF_DEL for Delete filter 1088 * @entry: In case of ADD filter, the caller passes the RQ number in this 1089 * variable. 1090 * This function stores the filter_id returned by the 1091 * firmware in the same variable before return; 1092 * 1093 * In case of DEL filter, the caller passes the RQ number. Return 1094 * value is irrelevant. 1095 * @data: filter data 1096 * @action: action data 1097 */ 1098 int vnic_dev_classifier(struct vnic_dev *vdev, u8 cmd, u16 *entry, 1099 struct filter_v2 *data, struct filter_action_v2 *action_v2) 1100 { 1101 u64 a0 = 0, a1 = 0; 1102 int wait = 1000; 1103 dma_addr_t tlv_pa; 1104 int ret = -EINVAL; 1105 struct filter_tlv *tlv, *tlv_va; 1106 u64 tlv_size; 1107 u32 filter_size, action_size; 1108 static unsigned int unique_id; 1109 char z_name[RTE_MEMZONE_NAMESIZE]; 1110 enum vnic_devcmd_cmd dev_cmd; 1111 1112 if (cmd == CLSF_ADD) { 1113 dev_cmd = (data->type >= FILTER_DPDK_1) ? 1114 CMD_ADD_ADV_FILTER : CMD_ADD_FILTER; 1115 1116 filter_size = vnic_filter_size(data); 1117 action_size = vnic_action_size(action_v2); 1118 1119 tlv_size = filter_size + action_size + 1120 2*sizeof(struct filter_tlv); 1121 snprintf((char *)z_name, sizeof(z_name), 1122 "vnic_clsf_%u", unique_id++); 1123 tlv_va = vdev->alloc_consistent(vdev->priv, 1124 tlv_size, &tlv_pa, (u8 *)z_name); 1125 if (!tlv_va) 1126 return -ENOMEM; 1127 tlv = tlv_va; 1128 a0 = tlv_pa; 1129 a1 = tlv_size; 1130 memset(tlv, 0, tlv_size); 1131 tlv->type = CLSF_TLV_FILTER; 1132 tlv->length = filter_size; 1133 memcpy(&tlv->val, (void *)data, filter_size); 1134 1135 tlv = (struct filter_tlv *)((char *)tlv + 1136 sizeof(struct filter_tlv) + 1137 filter_size); 1138 1139 tlv->type = CLSF_TLV_ACTION; 1140 tlv->length = action_size; 1141 memcpy(&tlv->val, (void *)action_v2, action_size); 1142 ret = vnic_dev_cmd(vdev, dev_cmd, &a0, &a1, wait); 1143 *entry = (u16)a0; 1144 vdev->free_consistent(vdev->priv, tlv_size, tlv_va, tlv_pa); 1145 } else if (cmd == CLSF_DEL) { 1146 a0 = *entry; 1147 ret = vnic_dev_cmd(vdev, CMD_DEL_FILTER, &a0, &a1, wait); 1148 } 1149 1150 return ret; 1151 } 1152 1153 int vnic_dev_overlay_offload_ctrl(struct vnic_dev *vdev, u8 overlay, u8 config) 1154 { 1155 u64 a0 = overlay; 1156 u64 a1 = config; 1157 int wait = 1000; 1158 1159 return vnic_dev_cmd(vdev, CMD_OVERLAY_OFFLOAD_CTRL, &a0, &a1, wait); 1160 } 1161 1162 int vnic_dev_overlay_offload_cfg(struct vnic_dev *vdev, u8 overlay, 1163 u16 vxlan_udp_port_number) 1164 { 1165 u64 a1 = vxlan_udp_port_number; 1166 u64 a0 = overlay; 1167 int wait = 1000; 1168 1169 return vnic_dev_cmd(vdev, CMD_OVERLAY_OFFLOAD_CFG, &a0, &a1, wait); 1170 } 1171 1172 int vnic_dev_capable_vxlan(struct vnic_dev *vdev) 1173 { 1174 u64 a0 = VIC_FEATURE_VXLAN; 1175 u64 a1 = 0; 1176 int wait = 1000; 1177 int ret; 1178 1179 ret = vnic_dev_cmd(vdev, CMD_GET_SUPP_FEATURE_VER, &a0, &a1, wait); 1180 /* 1 if the NIC can do VXLAN for both IPv4 and IPv6 with multiple WQs */ 1181 return ret == 0 && 1182 (a1 & (FEATURE_VXLAN_IPV6 | FEATURE_VXLAN_MULTI_WQ)) == 1183 (FEATURE_VXLAN_IPV6 | FEATURE_VXLAN_MULTI_WQ); 1184 } 1185 1186 bool vnic_dev_counter_alloc(struct vnic_dev *vdev, uint32_t *idx) 1187 { 1188 u64 a0 = 0; 1189 u64 a1 = 0; 1190 int wait = 1000; 1191 1192 if (vnic_dev_cmd(vdev, CMD_COUNTER_ALLOC, &a0, &a1, wait)) 1193 return false; 1194 *idx = (uint32_t)a0; 1195 return true; 1196 } 1197 1198 bool vnic_dev_counter_free(struct vnic_dev *vdev, uint32_t idx) 1199 { 1200 u64 a0 = idx; 1201 u64 a1 = 0; 1202 int wait = 1000; 1203 1204 return vnic_dev_cmd(vdev, CMD_COUNTER_FREE, &a0, &a1, 1205 wait) == 0; 1206 } 1207 1208 bool vnic_dev_counter_query(struct vnic_dev *vdev, uint32_t idx, 1209 bool reset, uint64_t *packets, uint64_t *bytes) 1210 { 1211 u64 a0 = idx; 1212 u64 a1 = reset ? 1 : 0; 1213 int wait = 1000; 1214 1215 if (reset) { 1216 /* query/reset returns updated counters */ 1217 if (vnic_dev_cmd(vdev, CMD_COUNTER_QUERY, &a0, &a1, wait)) 1218 return false; 1219 *packets = a0; 1220 *bytes = a1; 1221 } else { 1222 /* Get values DMA'd from the adapter */ 1223 *packets = vdev->flow_counters[idx].vcc_packets; 1224 *bytes = vdev->flow_counters[idx].vcc_bytes; 1225 } 1226 return true; 1227 } 1228