1 /* 2 * BSD LICENSE 3 * 4 * Copyright(c) 2017 Cavium, Inc.. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Cavium, Inc. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <rte_ethdev.h> 35 #include <rte_cycles.h> 36 #include <rte_malloc.h> 37 38 #include "lio_logs.h" 39 #include "lio_struct.h" 40 #include "lio_ethdev.h" 41 #include "lio_rxtx.h" 42 43 #define LIO_MAX_SG 12 44 /* Flush iq if available tx_desc fall below LIO_FLUSH_WM */ 45 #define LIO_FLUSH_WM(_iq) ((_iq)->max_count / 2) 46 #define LIO_PKT_IN_DONE_CNT_MASK 0x00000000FFFFFFFFULL 47 48 static void 49 lio_droq_compute_max_packet_bufs(struct lio_droq *droq) 50 { 51 uint32_t count = 0; 52 53 do { 54 count += droq->buffer_size; 55 } while (count < LIO_MAX_RX_PKTLEN); 56 } 57 58 static void 59 lio_droq_reset_indices(struct lio_droq *droq) 60 { 61 droq->read_idx = 0; 62 droq->write_idx = 0; 63 droq->refill_idx = 0; 64 droq->refill_count = 0; 65 rte_atomic64_set(&droq->pkts_pending, 0); 66 } 67 68 static void 69 lio_droq_destroy_ring_buffers(struct lio_droq *droq) 70 { 71 uint32_t i; 72 73 for (i = 0; i < droq->max_count; i++) { 74 if (droq->recv_buf_list[i].buffer) { 75 rte_pktmbuf_free((struct rte_mbuf *) 76 droq->recv_buf_list[i].buffer); 77 droq->recv_buf_list[i].buffer = NULL; 78 } 79 } 80 81 lio_droq_reset_indices(droq); 82 } 83 84 static int 85 lio_droq_setup_ring_buffers(struct lio_device *lio_dev, 86 struct lio_droq *droq) 87 { 88 struct lio_droq_desc *desc_ring = droq->desc_ring; 89 uint32_t i; 90 void *buf; 91 92 for (i = 0; i < droq->max_count; i++) { 93 buf = rte_pktmbuf_alloc(droq->mpool); 94 if (buf == NULL) { 95 lio_dev_err(lio_dev, "buffer alloc failed\n"); 96 droq->stats.rx_alloc_failure++; 97 lio_droq_destroy_ring_buffers(droq); 98 return -ENOMEM; 99 } 100 101 droq->recv_buf_list[i].buffer = buf; 102 droq->info_list[i].length = 0; 103 104 /* map ring buffers into memory */ 105 desc_ring[i].info_ptr = lio_map_ring_info(droq, i); 106 desc_ring[i].buffer_ptr = 107 lio_map_ring(droq->recv_buf_list[i].buffer); 108 } 109 110 lio_droq_reset_indices(droq); 111 112 lio_droq_compute_max_packet_bufs(droq); 113 114 return 0; 115 } 116 117 static void 118 lio_dma_zone_free(struct lio_device *lio_dev, const struct rte_memzone *mz) 119 { 120 const struct rte_memzone *mz_tmp; 121 int ret = 0; 122 123 if (mz == NULL) { 124 lio_dev_err(lio_dev, "Memzone NULL\n"); 125 return; 126 } 127 128 mz_tmp = rte_memzone_lookup(mz->name); 129 if (mz_tmp == NULL) { 130 lio_dev_err(lio_dev, "Memzone %s Not Found\n", mz->name); 131 return; 132 } 133 134 ret = rte_memzone_free(mz); 135 if (ret) 136 lio_dev_err(lio_dev, "Memzone free Failed ret %d\n", ret); 137 } 138 139 /** 140 * Frees the space for descriptor ring for the droq. 141 * 142 * @param lio_dev - pointer to the lio device structure 143 * @param q_no - droq no. 144 */ 145 static void 146 lio_delete_droq(struct lio_device *lio_dev, uint32_t q_no) 147 { 148 struct lio_droq *droq = lio_dev->droq[q_no]; 149 150 lio_dev_dbg(lio_dev, "OQ[%d]\n", q_no); 151 152 lio_droq_destroy_ring_buffers(droq); 153 rte_free(droq->recv_buf_list); 154 droq->recv_buf_list = NULL; 155 lio_dma_zone_free(lio_dev, droq->info_mz); 156 lio_dma_zone_free(lio_dev, droq->desc_ring_mz); 157 158 memset(droq, 0, LIO_DROQ_SIZE); 159 } 160 161 static void * 162 lio_alloc_info_buffer(struct lio_device *lio_dev, 163 struct lio_droq *droq, unsigned int socket_id) 164 { 165 droq->info_mz = rte_eth_dma_zone_reserve(lio_dev->eth_dev, 166 "info_list", droq->q_no, 167 (droq->max_count * 168 LIO_DROQ_INFO_SIZE), 169 RTE_CACHE_LINE_SIZE, 170 socket_id); 171 172 if (droq->info_mz == NULL) 173 return NULL; 174 175 droq->info_list_dma = droq->info_mz->iova; 176 droq->info_alloc_size = droq->info_mz->len; 177 droq->info_base_addr = (size_t)droq->info_mz->addr; 178 179 return droq->info_mz->addr; 180 } 181 182 /** 183 * Allocates space for the descriptor ring for the droq and 184 * sets the base addr, num desc etc in Octeon registers. 185 * 186 * @param lio_dev - pointer to the lio device structure 187 * @param q_no - droq no. 188 * @param app_ctx - pointer to application context 189 * @return Success: 0 Failure: -1 190 */ 191 static int 192 lio_init_droq(struct lio_device *lio_dev, uint32_t q_no, 193 uint32_t num_descs, uint32_t desc_size, 194 struct rte_mempool *mpool, unsigned int socket_id) 195 { 196 uint32_t c_refill_threshold; 197 uint32_t desc_ring_size; 198 struct lio_droq *droq; 199 200 lio_dev_dbg(lio_dev, "OQ[%d]\n", q_no); 201 202 droq = lio_dev->droq[q_no]; 203 droq->lio_dev = lio_dev; 204 droq->q_no = q_no; 205 droq->mpool = mpool; 206 207 c_refill_threshold = LIO_OQ_REFILL_THRESHOLD_CFG(lio_dev); 208 209 droq->max_count = num_descs; 210 droq->buffer_size = desc_size; 211 212 desc_ring_size = droq->max_count * LIO_DROQ_DESC_SIZE; 213 droq->desc_ring_mz = rte_eth_dma_zone_reserve(lio_dev->eth_dev, 214 "droq", q_no, 215 desc_ring_size, 216 RTE_CACHE_LINE_SIZE, 217 socket_id); 218 219 if (droq->desc_ring_mz == NULL) { 220 lio_dev_err(lio_dev, 221 "Output queue %d ring alloc failed\n", q_no); 222 return -1; 223 } 224 225 droq->desc_ring_dma = droq->desc_ring_mz->iova; 226 droq->desc_ring = (struct lio_droq_desc *)droq->desc_ring_mz->addr; 227 228 lio_dev_dbg(lio_dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n", 229 q_no, droq->desc_ring, (unsigned long)droq->desc_ring_dma); 230 lio_dev_dbg(lio_dev, "droq[%d]: num_desc: %d\n", q_no, 231 droq->max_count); 232 233 droq->info_list = lio_alloc_info_buffer(lio_dev, droq, socket_id); 234 if (droq->info_list == NULL) { 235 lio_dev_err(lio_dev, "Cannot allocate memory for info list.\n"); 236 goto init_droq_fail; 237 } 238 239 droq->recv_buf_list = rte_zmalloc_socket("recv_buf_list", 240 (droq->max_count * 241 LIO_DROQ_RECVBUF_SIZE), 242 RTE_CACHE_LINE_SIZE, 243 socket_id); 244 if (droq->recv_buf_list == NULL) { 245 lio_dev_err(lio_dev, 246 "Output queue recv buf list alloc failed\n"); 247 goto init_droq_fail; 248 } 249 250 if (lio_droq_setup_ring_buffers(lio_dev, droq)) 251 goto init_droq_fail; 252 253 droq->refill_threshold = c_refill_threshold; 254 255 rte_spinlock_init(&droq->lock); 256 257 lio_dev->fn_list.setup_oq_regs(lio_dev, q_no); 258 259 lio_dev->io_qmask.oq |= (1ULL << q_no); 260 261 return 0; 262 263 init_droq_fail: 264 lio_delete_droq(lio_dev, q_no); 265 266 return -1; 267 } 268 269 int 270 lio_setup_droq(struct lio_device *lio_dev, int oq_no, int num_descs, 271 int desc_size, struct rte_mempool *mpool, unsigned int socket_id) 272 { 273 struct lio_droq *droq; 274 275 PMD_INIT_FUNC_TRACE(); 276 277 if (lio_dev->droq[oq_no]) { 278 lio_dev_dbg(lio_dev, "Droq %d in use\n", oq_no); 279 return 0; 280 } 281 282 /* Allocate the DS for the new droq. */ 283 droq = rte_zmalloc_socket("ethdev RX queue", sizeof(*droq), 284 RTE_CACHE_LINE_SIZE, socket_id); 285 if (droq == NULL) 286 return -ENOMEM; 287 288 lio_dev->droq[oq_no] = droq; 289 290 /* Initialize the Droq */ 291 if (lio_init_droq(lio_dev, oq_no, num_descs, desc_size, mpool, 292 socket_id)) { 293 lio_dev_err(lio_dev, "Droq[%u] Initialization Failed\n", oq_no); 294 rte_free(lio_dev->droq[oq_no]); 295 lio_dev->droq[oq_no] = NULL; 296 return -ENOMEM; 297 } 298 299 lio_dev->num_oqs++; 300 301 lio_dev_dbg(lio_dev, "Total number of OQ: %d\n", lio_dev->num_oqs); 302 303 /* Send credit for octeon output queues. credits are always 304 * sent after the output queue is enabled. 305 */ 306 rte_write32(lio_dev->droq[oq_no]->max_count, 307 lio_dev->droq[oq_no]->pkts_credit_reg); 308 rte_wmb(); 309 310 return 0; 311 } 312 313 static inline uint32_t 314 lio_droq_get_bufcount(uint32_t buf_size, uint32_t total_len) 315 { 316 uint32_t buf_cnt = 0; 317 318 while (total_len > (buf_size * buf_cnt)) 319 buf_cnt++; 320 321 return buf_cnt; 322 } 323 324 /* If we were not able to refill all buffers, try to move around 325 * the buffers that were not dispatched. 326 */ 327 static inline uint32_t 328 lio_droq_refill_pullup_descs(struct lio_droq *droq, 329 struct lio_droq_desc *desc_ring) 330 { 331 uint32_t refill_index = droq->refill_idx; 332 uint32_t desc_refilled = 0; 333 334 while (refill_index != droq->read_idx) { 335 if (droq->recv_buf_list[refill_index].buffer) { 336 droq->recv_buf_list[droq->refill_idx].buffer = 337 droq->recv_buf_list[refill_index].buffer; 338 desc_ring[droq->refill_idx].buffer_ptr = 339 desc_ring[refill_index].buffer_ptr; 340 droq->recv_buf_list[refill_index].buffer = NULL; 341 desc_ring[refill_index].buffer_ptr = 0; 342 do { 343 droq->refill_idx = lio_incr_index( 344 droq->refill_idx, 1, 345 droq->max_count); 346 desc_refilled++; 347 droq->refill_count--; 348 } while (droq->recv_buf_list[droq->refill_idx].buffer); 349 } 350 refill_index = lio_incr_index(refill_index, 1, 351 droq->max_count); 352 } /* while */ 353 354 return desc_refilled; 355 } 356 357 /* lio_droq_refill 358 * 359 * @param droq - droq in which descriptors require new buffers. 360 * 361 * Description: 362 * Called during normal DROQ processing in interrupt mode or by the poll 363 * thread to refill the descriptors from which buffers were dispatched 364 * to upper layers. Attempts to allocate new buffers. If that fails, moves 365 * up buffers (that were not dispatched) to form a contiguous ring. 366 * 367 * Returns: 368 * No of descriptors refilled. 369 * 370 * Locks: 371 * This routine is called with droq->lock held. 372 */ 373 static uint32_t 374 lio_droq_refill(struct lio_droq *droq) 375 { 376 struct lio_droq_desc *desc_ring; 377 uint32_t desc_refilled = 0; 378 void *buf = NULL; 379 380 desc_ring = droq->desc_ring; 381 382 while (droq->refill_count && (desc_refilled < droq->max_count)) { 383 /* If a valid buffer exists (happens if there is no dispatch), 384 * reuse the buffer, else allocate. 385 */ 386 if (droq->recv_buf_list[droq->refill_idx].buffer == NULL) { 387 buf = rte_pktmbuf_alloc(droq->mpool); 388 /* If a buffer could not be allocated, no point in 389 * continuing 390 */ 391 if (buf == NULL) { 392 droq->stats.rx_alloc_failure++; 393 break; 394 } 395 396 droq->recv_buf_list[droq->refill_idx].buffer = buf; 397 } 398 399 desc_ring[droq->refill_idx].buffer_ptr = 400 lio_map_ring(droq->recv_buf_list[droq->refill_idx].buffer); 401 /* Reset any previous values in the length field. */ 402 droq->info_list[droq->refill_idx].length = 0; 403 404 droq->refill_idx = lio_incr_index(droq->refill_idx, 1, 405 droq->max_count); 406 desc_refilled++; 407 droq->refill_count--; 408 } 409 410 if (droq->refill_count) 411 desc_refilled += lio_droq_refill_pullup_descs(droq, desc_ring); 412 413 /* if droq->refill_count 414 * The refill count would not change in pass two. We only moved buffers 415 * to close the gap in the ring, but we would still have the same no. of 416 * buffers to refill. 417 */ 418 return desc_refilled; 419 } 420 421 static int 422 lio_droq_fast_process_packet(struct lio_device *lio_dev, 423 struct lio_droq *droq, 424 struct rte_mbuf **rx_pkts) 425 { 426 struct rte_mbuf *nicbuf = NULL; 427 struct lio_droq_info *info; 428 uint32_t total_len = 0; 429 int data_total_len = 0; 430 uint32_t pkt_len = 0; 431 union octeon_rh *rh; 432 int data_pkts = 0; 433 434 info = &droq->info_list[droq->read_idx]; 435 lio_swap_8B_data((uint64_t *)info, 2); 436 437 if (!info->length) 438 return -1; 439 440 /* Len of resp hdr in included in the received data len. */ 441 info->length -= OCTEON_RH_SIZE; 442 rh = &info->rh; 443 444 total_len += (uint32_t)info->length; 445 446 if (lio_opcode_slow_path(rh)) { 447 uint32_t buf_cnt; 448 449 buf_cnt = lio_droq_get_bufcount(droq->buffer_size, 450 (uint32_t)info->length); 451 droq->read_idx = lio_incr_index(droq->read_idx, buf_cnt, 452 droq->max_count); 453 droq->refill_count += buf_cnt; 454 } else { 455 if (info->length <= droq->buffer_size) { 456 if (rh->r_dh.has_hash) 457 pkt_len = (uint32_t)(info->length - 8); 458 else 459 pkt_len = (uint32_t)info->length; 460 461 nicbuf = droq->recv_buf_list[droq->read_idx].buffer; 462 droq->recv_buf_list[droq->read_idx].buffer = NULL; 463 droq->read_idx = lio_incr_index( 464 droq->read_idx, 1, 465 droq->max_count); 466 droq->refill_count++; 467 468 if (likely(nicbuf != NULL)) { 469 /* We don't have a way to pass flags yet */ 470 nicbuf->ol_flags = 0; 471 if (rh->r_dh.has_hash) { 472 uint64_t *hash_ptr; 473 474 nicbuf->ol_flags |= PKT_RX_RSS_HASH; 475 hash_ptr = rte_pktmbuf_mtod(nicbuf, 476 uint64_t *); 477 lio_swap_8B_data(hash_ptr, 1); 478 nicbuf->hash.rss = (uint32_t)*hash_ptr; 479 nicbuf->data_off += 8; 480 } 481 482 nicbuf->pkt_len = pkt_len; 483 nicbuf->data_len = pkt_len; 484 nicbuf->port = lio_dev->port_id; 485 /* Store the mbuf */ 486 rx_pkts[data_pkts++] = nicbuf; 487 data_total_len += pkt_len; 488 } 489 490 /* Prefetch buffer pointers when on a cache line 491 * boundary 492 */ 493 if ((droq->read_idx & 3) == 0) { 494 rte_prefetch0( 495 &droq->recv_buf_list[droq->read_idx]); 496 rte_prefetch0( 497 &droq->info_list[droq->read_idx]); 498 } 499 } else { 500 struct rte_mbuf *first_buf = NULL; 501 struct rte_mbuf *last_buf = NULL; 502 503 while (pkt_len < info->length) { 504 int cpy_len = 0; 505 506 cpy_len = ((pkt_len + droq->buffer_size) > 507 info->length) 508 ? ((uint32_t)info->length - 509 pkt_len) 510 : droq->buffer_size; 511 512 nicbuf = 513 droq->recv_buf_list[droq->read_idx].buffer; 514 droq->recv_buf_list[droq->read_idx].buffer = 515 NULL; 516 517 if (likely(nicbuf != NULL)) { 518 /* Note the first seg */ 519 if (!pkt_len) 520 first_buf = nicbuf; 521 522 nicbuf->port = lio_dev->port_id; 523 /* We don't have a way to pass 524 * flags yet 525 */ 526 nicbuf->ol_flags = 0; 527 if ((!pkt_len) && (rh->r_dh.has_hash)) { 528 uint64_t *hash_ptr; 529 530 nicbuf->ol_flags |= 531 PKT_RX_RSS_HASH; 532 hash_ptr = rte_pktmbuf_mtod( 533 nicbuf, uint64_t *); 534 lio_swap_8B_data(hash_ptr, 1); 535 nicbuf->hash.rss = 536 (uint32_t)*hash_ptr; 537 nicbuf->data_off += 8; 538 nicbuf->pkt_len = cpy_len - 8; 539 nicbuf->data_len = cpy_len - 8; 540 } else { 541 nicbuf->pkt_len = cpy_len; 542 nicbuf->data_len = cpy_len; 543 } 544 545 if (pkt_len) 546 first_buf->nb_segs++; 547 548 if (last_buf) 549 last_buf->next = nicbuf; 550 551 last_buf = nicbuf; 552 } else { 553 PMD_RX_LOG(lio_dev, ERR, "no buf\n"); 554 } 555 556 pkt_len += cpy_len; 557 droq->read_idx = lio_incr_index( 558 droq->read_idx, 559 1, droq->max_count); 560 droq->refill_count++; 561 562 /* Prefetch buffer pointers when on a 563 * cache line boundary 564 */ 565 if ((droq->read_idx & 3) == 0) { 566 rte_prefetch0(&droq->recv_buf_list 567 [droq->read_idx]); 568 569 rte_prefetch0( 570 &droq->info_list[droq->read_idx]); 571 } 572 } 573 rx_pkts[data_pkts++] = first_buf; 574 if (rh->r_dh.has_hash) 575 data_total_len += (pkt_len - 8); 576 else 577 data_total_len += pkt_len; 578 } 579 580 /* Inform upper layer about packet checksum verification */ 581 struct rte_mbuf *m = rx_pkts[data_pkts - 1]; 582 583 if (rh->r_dh.csum_verified & LIO_IP_CSUM_VERIFIED) 584 m->ol_flags |= PKT_RX_IP_CKSUM_GOOD; 585 586 if (rh->r_dh.csum_verified & LIO_L4_CSUM_VERIFIED) 587 m->ol_flags |= PKT_RX_L4_CKSUM_GOOD; 588 } 589 590 if (droq->refill_count >= droq->refill_threshold) { 591 int desc_refilled = lio_droq_refill(droq); 592 593 /* Flush the droq descriptor data to memory to be sure 594 * that when we update the credits the data in memory is 595 * accurate. 596 */ 597 rte_wmb(); 598 rte_write32(desc_refilled, droq->pkts_credit_reg); 599 /* make sure mmio write completes */ 600 rte_wmb(); 601 } 602 603 info->length = 0; 604 info->rh.rh64 = 0; 605 606 droq->stats.pkts_received++; 607 droq->stats.rx_pkts_received += data_pkts; 608 droq->stats.rx_bytes_received += data_total_len; 609 droq->stats.bytes_received += total_len; 610 611 return data_pkts; 612 } 613 614 static uint32_t 615 lio_droq_fast_process_packets(struct lio_device *lio_dev, 616 struct lio_droq *droq, 617 struct rte_mbuf **rx_pkts, 618 uint32_t pkts_to_process) 619 { 620 int ret, data_pkts = 0; 621 uint32_t pkt; 622 623 for (pkt = 0; pkt < pkts_to_process; pkt++) { 624 ret = lio_droq_fast_process_packet(lio_dev, droq, 625 &rx_pkts[data_pkts]); 626 if (ret < 0) { 627 lio_dev_err(lio_dev, "Port[%d] DROQ[%d] idx: %d len:0, pkt_cnt: %d\n", 628 lio_dev->port_id, droq->q_no, 629 droq->read_idx, pkts_to_process); 630 break; 631 } 632 data_pkts += ret; 633 } 634 635 rte_atomic64_sub(&droq->pkts_pending, pkt); 636 637 return data_pkts; 638 } 639 640 static inline uint32_t 641 lio_droq_check_hw_for_pkts(struct lio_droq *droq) 642 { 643 uint32_t last_count; 644 uint32_t pkt_count; 645 646 pkt_count = rte_read32(droq->pkts_sent_reg); 647 648 last_count = pkt_count - droq->pkt_count; 649 droq->pkt_count = pkt_count; 650 651 if (last_count) 652 rte_atomic64_add(&droq->pkts_pending, last_count); 653 654 return last_count; 655 } 656 657 uint16_t 658 lio_dev_recv_pkts(void *rx_queue, 659 struct rte_mbuf **rx_pkts, 660 uint16_t budget) 661 { 662 struct lio_droq *droq = rx_queue; 663 struct lio_device *lio_dev = droq->lio_dev; 664 uint32_t pkts_processed = 0; 665 uint32_t pkt_count = 0; 666 667 lio_droq_check_hw_for_pkts(droq); 668 669 pkt_count = rte_atomic64_read(&droq->pkts_pending); 670 if (!pkt_count) 671 return 0; 672 673 if (pkt_count > budget) 674 pkt_count = budget; 675 676 /* Grab the lock */ 677 rte_spinlock_lock(&droq->lock); 678 pkts_processed = lio_droq_fast_process_packets(lio_dev, 679 droq, rx_pkts, 680 pkt_count); 681 682 if (droq->pkt_count) { 683 rte_write32(droq->pkt_count, droq->pkts_sent_reg); 684 droq->pkt_count = 0; 685 } 686 687 /* Release the spin lock */ 688 rte_spinlock_unlock(&droq->lock); 689 690 return pkts_processed; 691 } 692 693 void 694 lio_delete_droq_queue(struct lio_device *lio_dev, 695 int oq_no) 696 { 697 lio_delete_droq(lio_dev, oq_no); 698 lio_dev->num_oqs--; 699 rte_free(lio_dev->droq[oq_no]); 700 lio_dev->droq[oq_no] = NULL; 701 } 702 703 /** 704 * lio_init_instr_queue() 705 * @param lio_dev - pointer to the lio device structure. 706 * @param txpciq - queue to be initialized. 707 * 708 * Called at driver init time for each input queue. iq_conf has the 709 * configuration parameters for the queue. 710 * 711 * @return Success: 0 Failure: -1 712 */ 713 static int 714 lio_init_instr_queue(struct lio_device *lio_dev, 715 union octeon_txpciq txpciq, 716 uint32_t num_descs, unsigned int socket_id) 717 { 718 uint32_t iq_no = (uint32_t)txpciq.s.q_no; 719 struct lio_instr_queue *iq; 720 uint32_t instr_type; 721 uint32_t q_size; 722 723 instr_type = LIO_IQ_INSTR_TYPE(lio_dev); 724 725 q_size = instr_type * num_descs; 726 iq = lio_dev->instr_queue[iq_no]; 727 iq->iq_mz = rte_eth_dma_zone_reserve(lio_dev->eth_dev, 728 "instr_queue", iq_no, q_size, 729 RTE_CACHE_LINE_SIZE, 730 socket_id); 731 if (iq->iq_mz == NULL) { 732 lio_dev_err(lio_dev, "Cannot allocate memory for instr queue %d\n", 733 iq_no); 734 return -1; 735 } 736 737 iq->base_addr_dma = iq->iq_mz->iova; 738 iq->base_addr = (uint8_t *)iq->iq_mz->addr; 739 740 iq->max_count = num_descs; 741 742 /* Initialize a list to holds requests that have been posted to Octeon 743 * but has yet to be fetched by octeon 744 */ 745 iq->request_list = rte_zmalloc_socket("request_list", 746 sizeof(*iq->request_list) * 747 num_descs, 748 RTE_CACHE_LINE_SIZE, 749 socket_id); 750 if (iq->request_list == NULL) { 751 lio_dev_err(lio_dev, "Alloc failed for IQ[%d] nr free list\n", 752 iq_no); 753 lio_dma_zone_free(lio_dev, iq->iq_mz); 754 return -1; 755 } 756 757 lio_dev_dbg(lio_dev, "IQ[%d]: base: %p basedma: %lx count: %d\n", 758 iq_no, iq->base_addr, (unsigned long)iq->base_addr_dma, 759 iq->max_count); 760 761 iq->lio_dev = lio_dev; 762 iq->txpciq.txpciq64 = txpciq.txpciq64; 763 iq->fill_cnt = 0; 764 iq->host_write_index = 0; 765 iq->lio_read_index = 0; 766 iq->flush_index = 0; 767 768 rte_atomic64_set(&iq->instr_pending, 0); 769 770 /* Initialize the spinlock for this instruction queue */ 771 rte_spinlock_init(&iq->lock); 772 rte_spinlock_init(&iq->post_lock); 773 774 rte_atomic64_clear(&iq->iq_flush_running); 775 776 lio_dev->io_qmask.iq |= (1ULL << iq_no); 777 778 /* Set the 32B/64B mode for each input queue */ 779 lio_dev->io_qmask.iq64B |= ((instr_type == 64) << iq_no); 780 iq->iqcmd_64B = (instr_type == 64); 781 782 lio_dev->fn_list.setup_iq_regs(lio_dev, iq_no); 783 784 return 0; 785 } 786 787 int 788 lio_setup_instr_queue0(struct lio_device *lio_dev) 789 { 790 union octeon_txpciq txpciq; 791 uint32_t num_descs = 0; 792 uint32_t iq_no = 0; 793 794 num_descs = LIO_NUM_DEF_TX_DESCS_CFG(lio_dev); 795 796 lio_dev->num_iqs = 0; 797 798 lio_dev->instr_queue[0] = rte_zmalloc(NULL, 799 sizeof(struct lio_instr_queue), 0); 800 if (lio_dev->instr_queue[0] == NULL) 801 return -ENOMEM; 802 803 lio_dev->instr_queue[0]->q_index = 0; 804 lio_dev->instr_queue[0]->app_ctx = (void *)(size_t)0; 805 txpciq.txpciq64 = 0; 806 txpciq.s.q_no = iq_no; 807 txpciq.s.pkind = lio_dev->pfvf_hsword.pkind; 808 txpciq.s.use_qpg = 0; 809 txpciq.s.qpg = 0; 810 if (lio_init_instr_queue(lio_dev, txpciq, num_descs, SOCKET_ID_ANY)) { 811 rte_free(lio_dev->instr_queue[0]); 812 lio_dev->instr_queue[0] = NULL; 813 return -1; 814 } 815 816 lio_dev->num_iqs++; 817 818 return 0; 819 } 820 821 /** 822 * lio_delete_instr_queue() 823 * @param lio_dev - pointer to the lio device structure. 824 * @param iq_no - queue to be deleted. 825 * 826 * Called at driver unload time for each input queue. Deletes all 827 * allocated resources for the input queue. 828 */ 829 static void 830 lio_delete_instr_queue(struct lio_device *lio_dev, uint32_t iq_no) 831 { 832 struct lio_instr_queue *iq = lio_dev->instr_queue[iq_no]; 833 834 rte_free(iq->request_list); 835 iq->request_list = NULL; 836 lio_dma_zone_free(lio_dev, iq->iq_mz); 837 } 838 839 void 840 lio_free_instr_queue0(struct lio_device *lio_dev) 841 { 842 lio_delete_instr_queue(lio_dev, 0); 843 rte_free(lio_dev->instr_queue[0]); 844 lio_dev->instr_queue[0] = NULL; 845 lio_dev->num_iqs--; 846 } 847 848 /* Return 0 on success, -1 on failure */ 849 int 850 lio_setup_iq(struct lio_device *lio_dev, int q_index, 851 union octeon_txpciq txpciq, uint32_t num_descs, void *app_ctx, 852 unsigned int socket_id) 853 { 854 uint32_t iq_no = (uint32_t)txpciq.s.q_no; 855 856 if (lio_dev->instr_queue[iq_no]) { 857 lio_dev_dbg(lio_dev, "IQ is in use. Cannot create the IQ: %d again\n", 858 iq_no); 859 lio_dev->instr_queue[iq_no]->txpciq.txpciq64 = txpciq.txpciq64; 860 lio_dev->instr_queue[iq_no]->app_ctx = app_ctx; 861 return 0; 862 } 863 864 lio_dev->instr_queue[iq_no] = rte_zmalloc_socket("ethdev TX queue", 865 sizeof(struct lio_instr_queue), 866 RTE_CACHE_LINE_SIZE, socket_id); 867 if (lio_dev->instr_queue[iq_no] == NULL) 868 return -1; 869 870 lio_dev->instr_queue[iq_no]->q_index = q_index; 871 lio_dev->instr_queue[iq_no]->app_ctx = app_ctx; 872 873 if (lio_init_instr_queue(lio_dev, txpciq, num_descs, socket_id)) 874 goto release_lio_iq; 875 876 lio_dev->num_iqs++; 877 if (lio_dev->fn_list.enable_io_queues(lio_dev)) 878 goto delete_lio_iq; 879 880 return 0; 881 882 delete_lio_iq: 883 lio_delete_instr_queue(lio_dev, iq_no); 884 lio_dev->num_iqs--; 885 release_lio_iq: 886 rte_free(lio_dev->instr_queue[iq_no]); 887 lio_dev->instr_queue[iq_no] = NULL; 888 889 return -1; 890 } 891 892 int 893 lio_wait_for_instr_fetch(struct lio_device *lio_dev) 894 { 895 int pending, instr_cnt; 896 int i, retry = 1000; 897 898 do { 899 instr_cnt = 0; 900 901 for (i = 0; i < LIO_MAX_INSTR_QUEUES(lio_dev); i++) { 902 if (!(lio_dev->io_qmask.iq & (1ULL << i))) 903 continue; 904 905 if (lio_dev->instr_queue[i] == NULL) 906 break; 907 908 pending = rte_atomic64_read( 909 &lio_dev->instr_queue[i]->instr_pending); 910 if (pending) 911 lio_flush_iq(lio_dev, lio_dev->instr_queue[i]); 912 913 instr_cnt += pending; 914 } 915 916 if (instr_cnt == 0) 917 break; 918 919 rte_delay_ms(1); 920 921 } while (retry-- && instr_cnt); 922 923 return instr_cnt; 924 } 925 926 static inline void 927 lio_ring_doorbell(struct lio_device *lio_dev, 928 struct lio_instr_queue *iq) 929 { 930 if (rte_atomic64_read(&lio_dev->status) == LIO_DEV_RUNNING) { 931 rte_write32(iq->fill_cnt, iq->doorbell_reg); 932 /* make sure doorbell write goes through */ 933 rte_wmb(); 934 iq->fill_cnt = 0; 935 } 936 } 937 938 static inline void 939 copy_cmd_into_iq(struct lio_instr_queue *iq, uint8_t *cmd) 940 { 941 uint8_t *iqptr, cmdsize; 942 943 cmdsize = ((iq->iqcmd_64B) ? 64 : 32); 944 iqptr = iq->base_addr + (cmdsize * iq->host_write_index); 945 946 rte_memcpy(iqptr, cmd, cmdsize); 947 } 948 949 static inline struct lio_iq_post_status 950 post_command2(struct lio_instr_queue *iq, uint8_t *cmd) 951 { 952 struct lio_iq_post_status st; 953 954 st.status = LIO_IQ_SEND_OK; 955 956 /* This ensures that the read index does not wrap around to the same 957 * position if queue gets full before Octeon could fetch any instr. 958 */ 959 if (rte_atomic64_read(&iq->instr_pending) >= 960 (int32_t)(iq->max_count - 1)) { 961 st.status = LIO_IQ_SEND_FAILED; 962 st.index = -1; 963 return st; 964 } 965 966 if (rte_atomic64_read(&iq->instr_pending) >= 967 (int32_t)(iq->max_count - 2)) 968 st.status = LIO_IQ_SEND_STOP; 969 970 copy_cmd_into_iq(iq, cmd); 971 972 /* "index" is returned, host_write_index is modified. */ 973 st.index = iq->host_write_index; 974 iq->host_write_index = lio_incr_index(iq->host_write_index, 1, 975 iq->max_count); 976 iq->fill_cnt++; 977 978 /* Flush the command into memory. We need to be sure the data is in 979 * memory before indicating that the instruction is pending. 980 */ 981 rte_wmb(); 982 983 rte_atomic64_inc(&iq->instr_pending); 984 985 return st; 986 } 987 988 static inline void 989 lio_add_to_request_list(struct lio_instr_queue *iq, 990 int idx, void *buf, int reqtype) 991 { 992 iq->request_list[idx].buf = buf; 993 iq->request_list[idx].reqtype = reqtype; 994 } 995 996 static inline void 997 lio_free_netsgbuf(void *buf) 998 { 999 struct lio_buf_free_info *finfo = buf; 1000 struct lio_device *lio_dev = finfo->lio_dev; 1001 struct rte_mbuf *m = finfo->mbuf; 1002 struct lio_gather *g = finfo->g; 1003 uint8_t iq = finfo->iq_no; 1004 1005 /* This will take care of multiple segments also */ 1006 rte_pktmbuf_free(m); 1007 1008 rte_spinlock_lock(&lio_dev->glist_lock[iq]); 1009 STAILQ_INSERT_TAIL(&lio_dev->glist_head[iq], &g->list, entries); 1010 rte_spinlock_unlock(&lio_dev->glist_lock[iq]); 1011 rte_free(finfo); 1012 } 1013 1014 /* Can only run in process context */ 1015 static int 1016 lio_process_iq_request_list(struct lio_device *lio_dev, 1017 struct lio_instr_queue *iq) 1018 { 1019 struct octeon_instr_irh *irh = NULL; 1020 uint32_t old = iq->flush_index; 1021 struct lio_soft_command *sc; 1022 uint32_t inst_count = 0; 1023 int reqtype; 1024 void *buf; 1025 1026 while (old != iq->lio_read_index) { 1027 reqtype = iq->request_list[old].reqtype; 1028 buf = iq->request_list[old].buf; 1029 1030 if (reqtype == LIO_REQTYPE_NONE) 1031 goto skip_this; 1032 1033 switch (reqtype) { 1034 case LIO_REQTYPE_NORESP_NET: 1035 rte_pktmbuf_free((struct rte_mbuf *)buf); 1036 break; 1037 case LIO_REQTYPE_NORESP_NET_SG: 1038 lio_free_netsgbuf(buf); 1039 break; 1040 case LIO_REQTYPE_SOFT_COMMAND: 1041 sc = buf; 1042 irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh; 1043 if (irh->rflag) { 1044 /* We're expecting a response from Octeon. 1045 * It's up to lio_process_ordered_list() to 1046 * process sc. Add sc to the ordered soft 1047 * command response list because we expect 1048 * a response from Octeon. 1049 */ 1050 rte_spinlock_lock(&lio_dev->response_list.lock); 1051 rte_atomic64_inc( 1052 &lio_dev->response_list.pending_req_count); 1053 STAILQ_INSERT_TAIL( 1054 &lio_dev->response_list.head, 1055 &sc->node, entries); 1056 rte_spinlock_unlock( 1057 &lio_dev->response_list.lock); 1058 } else { 1059 if (sc->callback) { 1060 /* This callback must not sleep */ 1061 sc->callback(LIO_REQUEST_DONE, 1062 sc->callback_arg); 1063 } 1064 } 1065 break; 1066 default: 1067 lio_dev_err(lio_dev, 1068 "Unknown reqtype: %d buf: %p at idx %d\n", 1069 reqtype, buf, old); 1070 } 1071 1072 iq->request_list[old].buf = NULL; 1073 iq->request_list[old].reqtype = 0; 1074 1075 skip_this: 1076 inst_count++; 1077 old = lio_incr_index(old, 1, iq->max_count); 1078 } 1079 1080 iq->flush_index = old; 1081 1082 return inst_count; 1083 } 1084 1085 static void 1086 lio_update_read_index(struct lio_instr_queue *iq) 1087 { 1088 uint32_t pkt_in_done = rte_read32(iq->inst_cnt_reg); 1089 uint32_t last_done; 1090 1091 last_done = pkt_in_done - iq->pkt_in_done; 1092 iq->pkt_in_done = pkt_in_done; 1093 1094 /* Add last_done and modulo with the IQ size to get new index */ 1095 iq->lio_read_index = (iq->lio_read_index + 1096 (uint32_t)(last_done & LIO_PKT_IN_DONE_CNT_MASK)) % 1097 iq->max_count; 1098 } 1099 1100 int 1101 lio_flush_iq(struct lio_device *lio_dev, struct lio_instr_queue *iq) 1102 { 1103 uint32_t tot_inst_processed = 0; 1104 uint32_t inst_processed = 0; 1105 int tx_done = 1; 1106 1107 if (rte_atomic64_test_and_set(&iq->iq_flush_running) == 0) 1108 return tx_done; 1109 1110 rte_spinlock_lock(&iq->lock); 1111 1112 lio_update_read_index(iq); 1113 1114 do { 1115 /* Process any outstanding IQ packets. */ 1116 if (iq->flush_index == iq->lio_read_index) 1117 break; 1118 1119 inst_processed = lio_process_iq_request_list(lio_dev, iq); 1120 1121 if (inst_processed) { 1122 rte_atomic64_sub(&iq->instr_pending, inst_processed); 1123 iq->stats.instr_processed += inst_processed; 1124 } 1125 1126 tot_inst_processed += inst_processed; 1127 inst_processed = 0; 1128 1129 } while (1); 1130 1131 rte_spinlock_unlock(&iq->lock); 1132 1133 rte_atomic64_clear(&iq->iq_flush_running); 1134 1135 return tx_done; 1136 } 1137 1138 static int 1139 lio_send_command(struct lio_device *lio_dev, uint32_t iq_no, void *cmd, 1140 void *buf, uint32_t datasize, uint32_t reqtype) 1141 { 1142 struct lio_instr_queue *iq = lio_dev->instr_queue[iq_no]; 1143 struct lio_iq_post_status st; 1144 1145 rte_spinlock_lock(&iq->post_lock); 1146 1147 st = post_command2(iq, cmd); 1148 1149 if (st.status != LIO_IQ_SEND_FAILED) { 1150 lio_add_to_request_list(iq, st.index, buf, reqtype); 1151 LIO_INCR_INSTRQUEUE_PKT_COUNT(lio_dev, iq_no, bytes_sent, 1152 datasize); 1153 LIO_INCR_INSTRQUEUE_PKT_COUNT(lio_dev, iq_no, instr_posted, 1); 1154 1155 lio_ring_doorbell(lio_dev, iq); 1156 } else { 1157 LIO_INCR_INSTRQUEUE_PKT_COUNT(lio_dev, iq_no, instr_dropped, 1); 1158 } 1159 1160 rte_spinlock_unlock(&iq->post_lock); 1161 1162 return st.status; 1163 } 1164 1165 void 1166 lio_prepare_soft_command(struct lio_device *lio_dev, 1167 struct lio_soft_command *sc, uint8_t opcode, 1168 uint8_t subcode, uint32_t irh_ossp, uint64_t ossp0, 1169 uint64_t ossp1) 1170 { 1171 struct octeon_instr_pki_ih3 *pki_ih3; 1172 struct octeon_instr_ih3 *ih3; 1173 struct octeon_instr_irh *irh; 1174 struct octeon_instr_rdp *rdp; 1175 1176 RTE_ASSERT(opcode <= 15); 1177 RTE_ASSERT(subcode <= 127); 1178 1179 ih3 = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3; 1180 1181 ih3->pkind = lio_dev->instr_queue[sc->iq_no]->txpciq.s.pkind; 1182 1183 pki_ih3 = (struct octeon_instr_pki_ih3 *)&sc->cmd.cmd3.pki_ih3; 1184 1185 pki_ih3->w = 1; 1186 pki_ih3->raw = 1; 1187 pki_ih3->utag = 1; 1188 pki_ih3->uqpg = lio_dev->instr_queue[sc->iq_no]->txpciq.s.use_qpg; 1189 pki_ih3->utt = 1; 1190 1191 pki_ih3->tag = LIO_CONTROL; 1192 pki_ih3->tagtype = OCTEON_ATOMIC_TAG; 1193 pki_ih3->qpg = lio_dev->instr_queue[sc->iq_no]->txpciq.s.qpg; 1194 pki_ih3->pm = 0x7; 1195 pki_ih3->sl = 8; 1196 1197 if (sc->datasize) 1198 ih3->dlengsz = sc->datasize; 1199 1200 irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh; 1201 irh->opcode = opcode; 1202 irh->subcode = subcode; 1203 1204 /* opcode/subcode specific parameters (ossp) */ 1205 irh->ossp = irh_ossp; 1206 sc->cmd.cmd3.ossp[0] = ossp0; 1207 sc->cmd.cmd3.ossp[1] = ossp1; 1208 1209 if (sc->rdatasize) { 1210 rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd3.rdp; 1211 rdp->pcie_port = lio_dev->pcie_port; 1212 rdp->rlen = sc->rdatasize; 1213 irh->rflag = 1; 1214 /* PKI IH3 */ 1215 ih3->fsz = OCTEON_SOFT_CMD_RESP_IH3; 1216 } else { 1217 irh->rflag = 0; 1218 /* PKI IH3 */ 1219 ih3->fsz = OCTEON_PCI_CMD_O3; 1220 } 1221 } 1222 1223 int 1224 lio_send_soft_command(struct lio_device *lio_dev, 1225 struct lio_soft_command *sc) 1226 { 1227 struct octeon_instr_ih3 *ih3; 1228 struct octeon_instr_irh *irh; 1229 uint32_t len = 0; 1230 1231 ih3 = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3; 1232 if (ih3->dlengsz) { 1233 RTE_ASSERT(sc->dmadptr); 1234 sc->cmd.cmd3.dptr = sc->dmadptr; 1235 } 1236 1237 irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh; 1238 if (irh->rflag) { 1239 RTE_ASSERT(sc->dmarptr); 1240 RTE_ASSERT(sc->status_word != NULL); 1241 *sc->status_word = LIO_COMPLETION_WORD_INIT; 1242 sc->cmd.cmd3.rptr = sc->dmarptr; 1243 } 1244 1245 len = (uint32_t)ih3->dlengsz; 1246 1247 if (sc->wait_time) 1248 sc->timeout = lio_uptime + sc->wait_time; 1249 1250 return lio_send_command(lio_dev, sc->iq_no, &sc->cmd, sc, len, 1251 LIO_REQTYPE_SOFT_COMMAND); 1252 } 1253 1254 int 1255 lio_setup_sc_buffer_pool(struct lio_device *lio_dev) 1256 { 1257 char sc_pool_name[RTE_MEMPOOL_NAMESIZE]; 1258 uint16_t buf_size; 1259 1260 buf_size = LIO_SOFT_COMMAND_BUFFER_SIZE + RTE_PKTMBUF_HEADROOM; 1261 snprintf(sc_pool_name, sizeof(sc_pool_name), 1262 "lio_sc_pool_%u", lio_dev->port_id); 1263 lio_dev->sc_buf_pool = rte_pktmbuf_pool_create(sc_pool_name, 1264 LIO_MAX_SOFT_COMMAND_BUFFERS, 1265 0, 0, buf_size, SOCKET_ID_ANY); 1266 return 0; 1267 } 1268 1269 void 1270 lio_free_sc_buffer_pool(struct lio_device *lio_dev) 1271 { 1272 rte_mempool_free(lio_dev->sc_buf_pool); 1273 } 1274 1275 struct lio_soft_command * 1276 lio_alloc_soft_command(struct lio_device *lio_dev, uint32_t datasize, 1277 uint32_t rdatasize, uint32_t ctxsize) 1278 { 1279 uint32_t offset = sizeof(struct lio_soft_command); 1280 struct lio_soft_command *sc; 1281 struct rte_mbuf *m; 1282 uint64_t dma_addr; 1283 1284 RTE_ASSERT((offset + datasize + rdatasize + ctxsize) <= 1285 LIO_SOFT_COMMAND_BUFFER_SIZE); 1286 1287 m = rte_pktmbuf_alloc(lio_dev->sc_buf_pool); 1288 if (m == NULL) { 1289 lio_dev_err(lio_dev, "Cannot allocate mbuf for sc\n"); 1290 return NULL; 1291 } 1292 1293 /* set rte_mbuf data size and there is only 1 segment */ 1294 m->pkt_len = LIO_SOFT_COMMAND_BUFFER_SIZE; 1295 m->data_len = LIO_SOFT_COMMAND_BUFFER_SIZE; 1296 1297 /* use rte_mbuf buffer for soft command */ 1298 sc = rte_pktmbuf_mtod(m, struct lio_soft_command *); 1299 memset(sc, 0, LIO_SOFT_COMMAND_BUFFER_SIZE); 1300 sc->size = LIO_SOFT_COMMAND_BUFFER_SIZE; 1301 sc->dma_addr = rte_mbuf_data_iova(m); 1302 sc->mbuf = m; 1303 1304 dma_addr = sc->dma_addr; 1305 1306 if (ctxsize) { 1307 sc->ctxptr = (uint8_t *)sc + offset; 1308 sc->ctxsize = ctxsize; 1309 } 1310 1311 /* Start data at 128 byte boundary */ 1312 offset = (offset + ctxsize + 127) & 0xffffff80; 1313 1314 if (datasize) { 1315 sc->virtdptr = (uint8_t *)sc + offset; 1316 sc->dmadptr = dma_addr + offset; 1317 sc->datasize = datasize; 1318 } 1319 1320 /* Start rdata at 128 byte boundary */ 1321 offset = (offset + datasize + 127) & 0xffffff80; 1322 1323 if (rdatasize) { 1324 RTE_ASSERT(rdatasize >= 16); 1325 sc->virtrptr = (uint8_t *)sc + offset; 1326 sc->dmarptr = dma_addr + offset; 1327 sc->rdatasize = rdatasize; 1328 sc->status_word = (uint64_t *)((uint8_t *)(sc->virtrptr) + 1329 rdatasize - 8); 1330 } 1331 1332 return sc; 1333 } 1334 1335 void 1336 lio_free_soft_command(struct lio_soft_command *sc) 1337 { 1338 rte_pktmbuf_free(sc->mbuf); 1339 } 1340 1341 void 1342 lio_setup_response_list(struct lio_device *lio_dev) 1343 { 1344 STAILQ_INIT(&lio_dev->response_list.head); 1345 rte_spinlock_init(&lio_dev->response_list.lock); 1346 rte_atomic64_set(&lio_dev->response_list.pending_req_count, 0); 1347 } 1348 1349 int 1350 lio_process_ordered_list(struct lio_device *lio_dev) 1351 { 1352 int resp_to_process = LIO_MAX_ORD_REQS_TO_PROCESS; 1353 struct lio_response_list *ordered_sc_list; 1354 struct lio_soft_command *sc; 1355 int request_complete = 0; 1356 uint64_t status64; 1357 uint32_t status; 1358 1359 ordered_sc_list = &lio_dev->response_list; 1360 1361 do { 1362 rte_spinlock_lock(&ordered_sc_list->lock); 1363 1364 if (STAILQ_EMPTY(&ordered_sc_list->head)) { 1365 /* ordered_sc_list is empty; there is 1366 * nothing to process 1367 */ 1368 rte_spinlock_unlock(&ordered_sc_list->lock); 1369 return -1; 1370 } 1371 1372 sc = LIO_STQUEUE_FIRST_ENTRY(&ordered_sc_list->head, 1373 struct lio_soft_command, node); 1374 1375 status = LIO_REQUEST_PENDING; 1376 1377 /* check if octeon has finished DMA'ing a response 1378 * to where rptr is pointing to 1379 */ 1380 status64 = *sc->status_word; 1381 1382 if (status64 != LIO_COMPLETION_WORD_INIT) { 1383 /* This logic ensures that all 64b have been written. 1384 * 1. check byte 0 for non-FF 1385 * 2. if non-FF, then swap result from BE to host order 1386 * 3. check byte 7 (swapped to 0) for non-FF 1387 * 4. if non-FF, use the low 32-bit status code 1388 * 5. if either byte 0 or byte 7 is FF, don't use status 1389 */ 1390 if ((status64 & 0xff) != 0xff) { 1391 lio_swap_8B_data(&status64, 1); 1392 if (((status64 & 0xff) != 0xff)) { 1393 /* retrieve 16-bit firmware status */ 1394 status = (uint32_t)(status64 & 1395 0xffffULL); 1396 if (status) { 1397 status = 1398 LIO_FIRMWARE_STATUS_CODE( 1399 status); 1400 } else { 1401 /* i.e. no error */ 1402 status = LIO_REQUEST_DONE; 1403 } 1404 } 1405 } 1406 } else if ((sc->timeout && lio_check_timeout(lio_uptime, 1407 sc->timeout))) { 1408 lio_dev_err(lio_dev, 1409 "cmd failed, timeout (%ld, %ld)\n", 1410 (long)lio_uptime, (long)sc->timeout); 1411 status = LIO_REQUEST_TIMEOUT; 1412 } 1413 1414 if (status != LIO_REQUEST_PENDING) { 1415 /* we have received a response or we have timed out. 1416 * remove node from linked list 1417 */ 1418 STAILQ_REMOVE(&ordered_sc_list->head, 1419 &sc->node, lio_stailq_node, entries); 1420 rte_atomic64_dec( 1421 &lio_dev->response_list.pending_req_count); 1422 rte_spinlock_unlock(&ordered_sc_list->lock); 1423 1424 if (sc->callback) 1425 sc->callback(status, sc->callback_arg); 1426 1427 request_complete++; 1428 } else { 1429 /* no response yet */ 1430 request_complete = 0; 1431 rte_spinlock_unlock(&ordered_sc_list->lock); 1432 } 1433 1434 /* If we hit the Max Ordered requests to process every loop, 1435 * we quit and let this function be invoked the next time 1436 * the poll thread runs to process the remaining requests. 1437 * This function can take up the entire CPU if there is 1438 * no upper limit to the requests processed. 1439 */ 1440 if (request_complete >= resp_to_process) 1441 break; 1442 } while (request_complete); 1443 1444 return 0; 1445 } 1446 1447 static inline struct lio_stailq_node * 1448 list_delete_first_node(struct lio_stailq_head *head) 1449 { 1450 struct lio_stailq_node *node; 1451 1452 if (STAILQ_EMPTY(head)) 1453 node = NULL; 1454 else 1455 node = STAILQ_FIRST(head); 1456 1457 if (node) 1458 STAILQ_REMOVE(head, node, lio_stailq_node, entries); 1459 1460 return node; 1461 } 1462 1463 void 1464 lio_delete_sglist(struct lio_instr_queue *txq) 1465 { 1466 struct lio_device *lio_dev = txq->lio_dev; 1467 int iq_no = txq->q_index; 1468 struct lio_gather *g; 1469 1470 if (lio_dev->glist_head == NULL) 1471 return; 1472 1473 do { 1474 g = (struct lio_gather *)list_delete_first_node( 1475 &lio_dev->glist_head[iq_no]); 1476 if (g) { 1477 if (g->sg) 1478 rte_free( 1479 (void *)((unsigned long)g->sg - g->adjust)); 1480 rte_free(g); 1481 } 1482 } while (g); 1483 } 1484 1485 /** 1486 * \brief Setup gather lists 1487 * @param lio per-network private data 1488 */ 1489 int 1490 lio_setup_sglists(struct lio_device *lio_dev, int iq_no, 1491 int fw_mapped_iq, int num_descs, unsigned int socket_id) 1492 { 1493 struct lio_gather *g; 1494 int i; 1495 1496 rte_spinlock_init(&lio_dev->glist_lock[iq_no]); 1497 1498 STAILQ_INIT(&lio_dev->glist_head[iq_no]); 1499 1500 for (i = 0; i < num_descs; i++) { 1501 g = rte_zmalloc_socket(NULL, sizeof(*g), RTE_CACHE_LINE_SIZE, 1502 socket_id); 1503 if (g == NULL) { 1504 lio_dev_err(lio_dev, 1505 "lio_gather memory allocation failed for qno %d\n", 1506 iq_no); 1507 break; 1508 } 1509 1510 g->sg_size = 1511 ((ROUNDUP4(LIO_MAX_SG) >> 2) * LIO_SG_ENTRY_SIZE); 1512 1513 g->sg = rte_zmalloc_socket(NULL, g->sg_size + 8, 1514 RTE_CACHE_LINE_SIZE, socket_id); 1515 if (g->sg == NULL) { 1516 lio_dev_err(lio_dev, 1517 "sg list memory allocation failed for qno %d\n", 1518 iq_no); 1519 rte_free(g); 1520 break; 1521 } 1522 1523 /* The gather component should be aligned on 64-bit boundary */ 1524 if (((unsigned long)g->sg) & 7) { 1525 g->adjust = 8 - (((unsigned long)g->sg) & 7); 1526 g->sg = 1527 (struct lio_sg_entry *)((unsigned long)g->sg + 1528 g->adjust); 1529 } 1530 1531 STAILQ_INSERT_TAIL(&lio_dev->glist_head[iq_no], &g->list, 1532 entries); 1533 } 1534 1535 if (i != num_descs) { 1536 lio_delete_sglist(lio_dev->instr_queue[fw_mapped_iq]); 1537 return -ENOMEM; 1538 } 1539 1540 return 0; 1541 } 1542 1543 void 1544 lio_delete_instruction_queue(struct lio_device *lio_dev, int iq_no) 1545 { 1546 lio_delete_instr_queue(lio_dev, iq_no); 1547 rte_free(lio_dev->instr_queue[iq_no]); 1548 lio_dev->instr_queue[iq_no] = NULL; 1549 lio_dev->num_iqs--; 1550 } 1551 1552 static inline uint32_t 1553 lio_iq_get_available(struct lio_device *lio_dev, uint32_t q_no) 1554 { 1555 return ((lio_dev->instr_queue[q_no]->max_count - 1) - 1556 (uint32_t)rte_atomic64_read( 1557 &lio_dev->instr_queue[q_no]->instr_pending)); 1558 } 1559 1560 static inline int 1561 lio_iq_is_full(struct lio_device *lio_dev, uint32_t q_no) 1562 { 1563 return ((uint32_t)rte_atomic64_read( 1564 &lio_dev->instr_queue[q_no]->instr_pending) >= 1565 (lio_dev->instr_queue[q_no]->max_count - 2)); 1566 } 1567 1568 static int 1569 lio_dev_cleanup_iq(struct lio_device *lio_dev, int iq_no) 1570 { 1571 struct lio_instr_queue *iq = lio_dev->instr_queue[iq_no]; 1572 uint32_t count = 10000; 1573 1574 while ((lio_iq_get_available(lio_dev, iq_no) < LIO_FLUSH_WM(iq)) && 1575 --count) 1576 lio_flush_iq(lio_dev, iq); 1577 1578 return count ? 0 : 1; 1579 } 1580 1581 static void 1582 lio_ctrl_cmd_callback(uint32_t status __rte_unused, void *sc_ptr) 1583 { 1584 struct lio_soft_command *sc = sc_ptr; 1585 struct lio_dev_ctrl_cmd *ctrl_cmd; 1586 struct lio_ctrl_pkt *ctrl_pkt; 1587 1588 ctrl_pkt = (struct lio_ctrl_pkt *)sc->ctxptr; 1589 ctrl_cmd = ctrl_pkt->ctrl_cmd; 1590 ctrl_cmd->cond = 1; 1591 1592 lio_free_soft_command(sc); 1593 } 1594 1595 static inline struct lio_soft_command * 1596 lio_alloc_ctrl_pkt_sc(struct lio_device *lio_dev, 1597 struct lio_ctrl_pkt *ctrl_pkt) 1598 { 1599 struct lio_soft_command *sc = NULL; 1600 uint32_t uddsize, datasize; 1601 uint32_t rdatasize; 1602 uint8_t *data; 1603 1604 uddsize = (uint32_t)(ctrl_pkt->ncmd.s.more * 8); 1605 1606 datasize = OCTEON_CMD_SIZE + uddsize; 1607 rdatasize = (ctrl_pkt->wait_time) ? 16 : 0; 1608 1609 sc = lio_alloc_soft_command(lio_dev, datasize, 1610 rdatasize, sizeof(struct lio_ctrl_pkt)); 1611 if (sc == NULL) 1612 return NULL; 1613 1614 rte_memcpy(sc->ctxptr, ctrl_pkt, sizeof(struct lio_ctrl_pkt)); 1615 1616 data = (uint8_t *)sc->virtdptr; 1617 1618 rte_memcpy(data, &ctrl_pkt->ncmd, OCTEON_CMD_SIZE); 1619 1620 lio_swap_8B_data((uint64_t *)data, OCTEON_CMD_SIZE >> 3); 1621 1622 if (uddsize) { 1623 /* Endian-Swap for UDD should have been done by caller. */ 1624 rte_memcpy(data + OCTEON_CMD_SIZE, ctrl_pkt->udd, uddsize); 1625 } 1626 1627 sc->iq_no = (uint32_t)ctrl_pkt->iq_no; 1628 1629 lio_prepare_soft_command(lio_dev, sc, 1630 LIO_OPCODE, LIO_OPCODE_CMD, 1631 0, 0, 0); 1632 1633 sc->callback = lio_ctrl_cmd_callback; 1634 sc->callback_arg = sc; 1635 sc->wait_time = ctrl_pkt->wait_time; 1636 1637 return sc; 1638 } 1639 1640 int 1641 lio_send_ctrl_pkt(struct lio_device *lio_dev, struct lio_ctrl_pkt *ctrl_pkt) 1642 { 1643 struct lio_soft_command *sc = NULL; 1644 int retval; 1645 1646 sc = lio_alloc_ctrl_pkt_sc(lio_dev, ctrl_pkt); 1647 if (sc == NULL) { 1648 lio_dev_err(lio_dev, "soft command allocation failed\n"); 1649 return -1; 1650 } 1651 1652 retval = lio_send_soft_command(lio_dev, sc); 1653 if (retval == LIO_IQ_SEND_FAILED) { 1654 lio_free_soft_command(sc); 1655 lio_dev_err(lio_dev, "Port: %d soft command: %d send failed status: %x\n", 1656 lio_dev->port_id, ctrl_pkt->ncmd.s.cmd, retval); 1657 return -1; 1658 } 1659 1660 return retval; 1661 } 1662 1663 /** Send data packet to the device 1664 * @param lio_dev - lio device pointer 1665 * @param ndata - control structure with queueing, and buffer information 1666 * 1667 * @returns IQ_FAILED if it failed to add to the input queue. IQ_STOP if it the 1668 * queue should be stopped, and LIO_IQ_SEND_OK if it sent okay. 1669 */ 1670 static inline int 1671 lio_send_data_pkt(struct lio_device *lio_dev, struct lio_data_pkt *ndata) 1672 { 1673 return lio_send_command(lio_dev, ndata->q_no, &ndata->cmd, 1674 ndata->buf, ndata->datasize, ndata->reqtype); 1675 } 1676 1677 uint16_t 1678 lio_dev_xmit_pkts(void *tx_queue, struct rte_mbuf **pkts, uint16_t nb_pkts) 1679 { 1680 struct lio_instr_queue *txq = tx_queue; 1681 union lio_cmd_setup cmdsetup; 1682 struct lio_device *lio_dev; 1683 struct lio_iq_stats *stats; 1684 struct lio_data_pkt ndata; 1685 int i, processed = 0; 1686 struct rte_mbuf *m; 1687 uint32_t tag = 0; 1688 int status = 0; 1689 int iq_no; 1690 1691 lio_dev = txq->lio_dev; 1692 iq_no = txq->txpciq.s.q_no; 1693 stats = &lio_dev->instr_queue[iq_no]->stats; 1694 1695 if (!lio_dev->intf_open || !lio_dev->linfo.link.s.link_up) { 1696 PMD_TX_LOG(lio_dev, ERR, "Transmit failed link_status : %d\n", 1697 lio_dev->linfo.link.s.link_up); 1698 goto xmit_failed; 1699 } 1700 1701 lio_dev_cleanup_iq(lio_dev, iq_no); 1702 1703 for (i = 0; i < nb_pkts; i++) { 1704 uint32_t pkt_len = 0; 1705 1706 m = pkts[i]; 1707 1708 /* Prepare the attributes for the data to be passed to BASE. */ 1709 memset(&ndata, 0, sizeof(struct lio_data_pkt)); 1710 1711 ndata.buf = m; 1712 1713 ndata.q_no = iq_no; 1714 if (lio_iq_is_full(lio_dev, ndata.q_no)) { 1715 stats->tx_iq_busy++; 1716 if (lio_dev_cleanup_iq(lio_dev, iq_no)) { 1717 PMD_TX_LOG(lio_dev, ERR, 1718 "Transmit failed iq:%d full\n", 1719 ndata.q_no); 1720 break; 1721 } 1722 } 1723 1724 cmdsetup.cmd_setup64 = 0; 1725 cmdsetup.s.iq_no = iq_no; 1726 1727 /* check checksum offload flags to form cmd */ 1728 if (m->ol_flags & PKT_TX_IP_CKSUM) 1729 cmdsetup.s.ip_csum = 1; 1730 1731 if (m->ol_flags & PKT_TX_OUTER_IP_CKSUM) 1732 cmdsetup.s.tnl_csum = 1; 1733 else if ((m->ol_flags & PKT_TX_TCP_CKSUM) || 1734 (m->ol_flags & PKT_TX_UDP_CKSUM)) 1735 cmdsetup.s.transport_csum = 1; 1736 1737 if (m->nb_segs == 1) { 1738 pkt_len = rte_pktmbuf_data_len(m); 1739 cmdsetup.s.u.datasize = pkt_len; 1740 lio_prepare_pci_cmd(lio_dev, &ndata.cmd, 1741 &cmdsetup, tag); 1742 ndata.cmd.cmd3.dptr = rte_mbuf_data_iova(m); 1743 ndata.reqtype = LIO_REQTYPE_NORESP_NET; 1744 } else { 1745 struct lio_buf_free_info *finfo; 1746 struct lio_gather *g; 1747 rte_iova_t phyaddr; 1748 int i, frags; 1749 1750 finfo = (struct lio_buf_free_info *)rte_malloc(NULL, 1751 sizeof(*finfo), 0); 1752 if (finfo == NULL) { 1753 PMD_TX_LOG(lio_dev, ERR, 1754 "free buffer alloc failed\n"); 1755 goto xmit_failed; 1756 } 1757 1758 rte_spinlock_lock(&lio_dev->glist_lock[iq_no]); 1759 g = (struct lio_gather *)list_delete_first_node( 1760 &lio_dev->glist_head[iq_no]); 1761 rte_spinlock_unlock(&lio_dev->glist_lock[iq_no]); 1762 if (g == NULL) { 1763 PMD_TX_LOG(lio_dev, ERR, 1764 "Transmit scatter gather: glist null!\n"); 1765 goto xmit_failed; 1766 } 1767 1768 cmdsetup.s.gather = 1; 1769 cmdsetup.s.u.gatherptrs = m->nb_segs; 1770 lio_prepare_pci_cmd(lio_dev, &ndata.cmd, 1771 &cmdsetup, tag); 1772 1773 memset(g->sg, 0, g->sg_size); 1774 g->sg[0].ptr[0] = rte_mbuf_data_iova(m); 1775 lio_add_sg_size(&g->sg[0], m->data_len, 0); 1776 pkt_len = m->data_len; 1777 finfo->mbuf = m; 1778 1779 /* First seg taken care above */ 1780 frags = m->nb_segs - 1; 1781 i = 1; 1782 m = m->next; 1783 while (frags--) { 1784 g->sg[(i >> 2)].ptr[(i & 3)] = 1785 rte_mbuf_data_iova(m); 1786 lio_add_sg_size(&g->sg[(i >> 2)], 1787 m->data_len, (i & 3)); 1788 pkt_len += m->data_len; 1789 i++; 1790 m = m->next; 1791 } 1792 1793 phyaddr = rte_mem_virt2iova(g->sg); 1794 if (phyaddr == RTE_BAD_IOVA) { 1795 PMD_TX_LOG(lio_dev, ERR, "bad phys addr\n"); 1796 goto xmit_failed; 1797 } 1798 1799 ndata.cmd.cmd3.dptr = phyaddr; 1800 ndata.reqtype = LIO_REQTYPE_NORESP_NET_SG; 1801 1802 finfo->g = g; 1803 finfo->lio_dev = lio_dev; 1804 finfo->iq_no = (uint64_t)iq_no; 1805 ndata.buf = finfo; 1806 } 1807 1808 ndata.datasize = pkt_len; 1809 1810 status = lio_send_data_pkt(lio_dev, &ndata); 1811 1812 if (unlikely(status == LIO_IQ_SEND_FAILED)) { 1813 PMD_TX_LOG(lio_dev, ERR, "send failed\n"); 1814 break; 1815 } 1816 1817 if (unlikely(status == LIO_IQ_SEND_STOP)) { 1818 PMD_TX_LOG(lio_dev, DEBUG, "iq full\n"); 1819 /* create space as iq is full */ 1820 lio_dev_cleanup_iq(lio_dev, iq_no); 1821 } 1822 1823 stats->tx_done++; 1824 stats->tx_tot_bytes += pkt_len; 1825 processed++; 1826 } 1827 1828 xmit_failed: 1829 stats->tx_dropped += (nb_pkts - processed); 1830 1831 return processed; 1832 } 1833 1834 void 1835 lio_dev_clear_queues(struct rte_eth_dev *eth_dev) 1836 { 1837 struct lio_instr_queue *txq; 1838 struct lio_droq *rxq; 1839 uint16_t i; 1840 1841 for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { 1842 txq = eth_dev->data->tx_queues[i]; 1843 if (txq != NULL) { 1844 lio_dev_tx_queue_release(txq); 1845 eth_dev->data->tx_queues[i] = NULL; 1846 } 1847 } 1848 1849 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { 1850 rxq = eth_dev->data->rx_queues[i]; 1851 if (rxq != NULL) { 1852 lio_dev_rx_queue_release(rxq); 1853 eth_dev->data->rx_queues[i] = NULL; 1854 } 1855 } 1856 } 1857