1 /* QLogic qed NIC Driver 2 * Copyright (c) 2015 QLogic Corporation 3 * 4 * This software is available under the terms of the GNU General Public License 5 * (GPL) Version 2, available from the file COPYING in the main directory of 6 * this source tree. 7 */ 8 9 #ifndef _QED_CHAIN_H 10 #define _QED_CHAIN_H 11 12 #include <linux/types.h> 13 #include <asm/byteorder.h> 14 #include <linux/kernel.h> 15 #include <linux/list.h> 16 #include <linux/slab.h> 17 #include <linux/qed/common_hsi.h> 18 19 enum qed_chain_mode { 20 /* Each Page contains a next pointer at its end */ 21 QED_CHAIN_MODE_NEXT_PTR, 22 23 /* Chain is a single page (next ptr) is unrequired */ 24 QED_CHAIN_MODE_SINGLE, 25 26 /* Page pointers are located in a side list */ 27 QED_CHAIN_MODE_PBL, 28 }; 29 30 enum qed_chain_use_mode { 31 QED_CHAIN_USE_TO_PRODUCE, /* Chain starts empty */ 32 QED_CHAIN_USE_TO_CONSUME, /* Chain starts full */ 33 QED_CHAIN_USE_TO_CONSUME_PRODUCE, /* Chain starts empty */ 34 }; 35 36 enum qed_chain_cnt_type { 37 /* The chain's size/prod/cons are kept in 16-bit variables */ 38 QED_CHAIN_CNT_TYPE_U16, 39 40 /* The chain's size/prod/cons are kept in 32-bit variables */ 41 QED_CHAIN_CNT_TYPE_U32, 42 }; 43 44 struct qed_chain_next { 45 struct regpair next_phys; 46 void *next_virt; 47 }; 48 49 struct qed_chain_pbl_u16 { 50 u16 prod_page_idx; 51 u16 cons_page_idx; 52 }; 53 54 struct qed_chain_pbl_u32 { 55 u32 prod_page_idx; 56 u32 cons_page_idx; 57 }; 58 59 struct qed_chain_pbl { 60 /* Base address of a pre-allocated buffer for pbl */ 61 dma_addr_t p_phys_table; 62 void *p_virt_table; 63 64 /* Table for keeping the virtual addresses of the chain pages, 65 * respectively to the physical addresses in the pbl table. 66 */ 67 void **pp_virt_addr_tbl; 68 69 /* Index to current used page by producer/consumer */ 70 union { 71 struct qed_chain_pbl_u16 pbl16; 72 struct qed_chain_pbl_u32 pbl32; 73 } u; 74 }; 75 76 struct qed_chain_u16 { 77 /* Cyclic index of next element to produce/consme */ 78 u16 prod_idx; 79 u16 cons_idx; 80 }; 81 82 struct qed_chain_u32 { 83 /* Cyclic index of next element to produce/consme */ 84 u32 prod_idx; 85 u32 cons_idx; 86 }; 87 88 struct qed_chain { 89 void *p_virt_addr; 90 dma_addr_t p_phys_addr; 91 void *p_prod_elem; 92 void *p_cons_elem; 93 94 enum qed_chain_mode mode; 95 enum qed_chain_use_mode intended_use; /* used to produce/consume */ 96 enum qed_chain_cnt_type cnt_type; 97 98 union { 99 struct qed_chain_u16 chain16; 100 struct qed_chain_u32 chain32; 101 } u; 102 103 u32 page_cnt; 104 105 /* Number of elements - capacity is for usable elements only, 106 * while size will contain total number of elements [for entire chain]. 107 */ 108 u32 capacity; 109 u32 size; 110 111 /* Elements information for fast calculations */ 112 u16 elem_per_page; 113 u16 elem_per_page_mask; 114 u16 elem_unusable; 115 u16 usable_per_page; 116 u16 elem_size; 117 u16 next_page_mask; 118 struct qed_chain_pbl pbl; 119 }; 120 121 #define QED_CHAIN_PBL_ENTRY_SIZE (8) 122 #define QED_CHAIN_PAGE_SIZE (0x1000) 123 #define ELEMS_PER_PAGE(elem_size) (QED_CHAIN_PAGE_SIZE / (elem_size)) 124 125 #define UNUSABLE_ELEMS_PER_PAGE(elem_size, mode) \ 126 ((mode == QED_CHAIN_MODE_NEXT_PTR) ? \ 127 (1 + ((sizeof(struct qed_chain_next) - 1) / \ 128 (elem_size))) : 0) 129 130 #define USABLE_ELEMS_PER_PAGE(elem_size, mode) \ 131 ((u32)(ELEMS_PER_PAGE(elem_size) - \ 132 UNUSABLE_ELEMS_PER_PAGE(elem_size, mode))) 133 134 #define QED_CHAIN_PAGE_CNT(elem_cnt, elem_size, mode) \ 135 DIV_ROUND_UP(elem_cnt, USABLE_ELEMS_PER_PAGE(elem_size, mode)) 136 137 #define is_chain_u16(p) ((p)->cnt_type == QED_CHAIN_CNT_TYPE_U16) 138 #define is_chain_u32(p) ((p)->cnt_type == QED_CHAIN_CNT_TYPE_U32) 139 140 /* Accessors */ 141 static inline u16 qed_chain_get_prod_idx(struct qed_chain *p_chain) 142 { 143 return p_chain->u.chain16.prod_idx; 144 } 145 146 static inline u16 qed_chain_get_cons_idx(struct qed_chain *p_chain) 147 { 148 return p_chain->u.chain16.cons_idx; 149 } 150 151 static inline u32 qed_chain_get_cons_idx_u32(struct qed_chain *p_chain) 152 { 153 return p_chain->u.chain32.cons_idx; 154 } 155 156 static inline u16 qed_chain_get_elem_left(struct qed_chain *p_chain) 157 { 158 u16 used; 159 160 used = (u16) (((u32)0x10000 + 161 (u32)p_chain->u.chain16.prod_idx) - 162 (u32)p_chain->u.chain16.cons_idx); 163 if (p_chain->mode == QED_CHAIN_MODE_NEXT_PTR) 164 used -= p_chain->u.chain16.prod_idx / p_chain->elem_per_page - 165 p_chain->u.chain16.cons_idx / p_chain->elem_per_page; 166 167 return (u16)(p_chain->capacity - used); 168 } 169 170 static inline u32 qed_chain_get_elem_left_u32(struct qed_chain *p_chain) 171 { 172 u32 used; 173 174 used = (u32) (((u64)0x100000000ULL + 175 (u64)p_chain->u.chain32.prod_idx) - 176 (u64)p_chain->u.chain32.cons_idx); 177 if (p_chain->mode == QED_CHAIN_MODE_NEXT_PTR) 178 used -= p_chain->u.chain32.prod_idx / p_chain->elem_per_page - 179 p_chain->u.chain32.cons_idx / p_chain->elem_per_page; 180 181 return p_chain->capacity - used; 182 } 183 184 static inline u16 qed_chain_get_usable_per_page(struct qed_chain *p_chain) 185 { 186 return p_chain->usable_per_page; 187 } 188 189 static inline u16 qed_chain_get_unusable_per_page(struct qed_chain *p_chain) 190 { 191 return p_chain->elem_unusable; 192 } 193 194 static inline u32 qed_chain_get_page_cnt(struct qed_chain *p_chain) 195 { 196 return p_chain->page_cnt; 197 } 198 199 static inline dma_addr_t qed_chain_get_pbl_phys(struct qed_chain *p_chain) 200 { 201 return p_chain->pbl.p_phys_table; 202 } 203 204 /** 205 * @brief qed_chain_advance_page - 206 * 207 * Advance the next element accros pages for a linked chain 208 * 209 * @param p_chain 210 * @param p_next_elem 211 * @param idx_to_inc 212 * @param page_to_inc 213 */ 214 static inline void 215 qed_chain_advance_page(struct qed_chain *p_chain, 216 void **p_next_elem, void *idx_to_inc, void *page_to_inc) 217 218 { 219 struct qed_chain_next *p_next = NULL; 220 u32 page_index = 0; 221 switch (p_chain->mode) { 222 case QED_CHAIN_MODE_NEXT_PTR: 223 p_next = *p_next_elem; 224 *p_next_elem = p_next->next_virt; 225 if (is_chain_u16(p_chain)) 226 *(u16 *)idx_to_inc += p_chain->elem_unusable; 227 else 228 *(u32 *)idx_to_inc += p_chain->elem_unusable; 229 break; 230 case QED_CHAIN_MODE_SINGLE: 231 *p_next_elem = p_chain->p_virt_addr; 232 break; 233 234 case QED_CHAIN_MODE_PBL: 235 if (is_chain_u16(p_chain)) { 236 if (++(*(u16 *)page_to_inc) == p_chain->page_cnt) 237 *(u16 *)page_to_inc = 0; 238 page_index = *(u16 *)page_to_inc; 239 } else { 240 if (++(*(u32 *)page_to_inc) == p_chain->page_cnt) 241 *(u32 *)page_to_inc = 0; 242 page_index = *(u32 *)page_to_inc; 243 } 244 *p_next_elem = p_chain->pbl.pp_virt_addr_tbl[page_index]; 245 } 246 } 247 248 #define is_unusable_idx(p, idx) \ 249 (((p)->u.chain16.idx & (p)->elem_per_page_mask) == (p)->usable_per_page) 250 251 #define is_unusable_idx_u32(p, idx) \ 252 (((p)->u.chain32.idx & (p)->elem_per_page_mask) == (p)->usable_per_page) 253 #define is_unusable_next_idx(p, idx) \ 254 ((((p)->u.chain16.idx + 1) & (p)->elem_per_page_mask) == \ 255 (p)->usable_per_page) 256 257 #define is_unusable_next_idx_u32(p, idx) \ 258 ((((p)->u.chain32.idx + 1) & (p)->elem_per_page_mask) == \ 259 (p)->usable_per_page) 260 261 #define test_and_skip(p, idx) \ 262 do { \ 263 if (is_chain_u16(p)) { \ 264 if (is_unusable_idx(p, idx)) \ 265 (p)->u.chain16.idx += (p)->elem_unusable; \ 266 } else { \ 267 if (is_unusable_idx_u32(p, idx)) \ 268 (p)->u.chain32.idx += (p)->elem_unusable; \ 269 } \ 270 } while (0) 271 272 /** 273 * @brief qed_chain_return_produced - 274 * 275 * A chain in which the driver "Produces" elements should use this API 276 * to indicate previous produced elements are now consumed. 277 * 278 * @param p_chain 279 */ 280 static inline void qed_chain_return_produced(struct qed_chain *p_chain) 281 { 282 if (is_chain_u16(p_chain)) 283 p_chain->u.chain16.cons_idx++; 284 else 285 p_chain->u.chain32.cons_idx++; 286 test_and_skip(p_chain, cons_idx); 287 } 288 289 /** 290 * @brief qed_chain_produce - 291 * 292 * A chain in which the driver "Produces" elements should use this to get 293 * a pointer to the next element which can be "Produced". It's driver 294 * responsibility to validate that the chain has room for new element. 295 * 296 * @param p_chain 297 * 298 * @return void*, a pointer to next element 299 */ 300 static inline void *qed_chain_produce(struct qed_chain *p_chain) 301 { 302 void *p_ret = NULL, *p_prod_idx, *p_prod_page_idx; 303 304 if (is_chain_u16(p_chain)) { 305 if ((p_chain->u.chain16.prod_idx & 306 p_chain->elem_per_page_mask) == p_chain->next_page_mask) { 307 p_prod_idx = &p_chain->u.chain16.prod_idx; 308 p_prod_page_idx = &p_chain->pbl.u.pbl16.prod_page_idx; 309 qed_chain_advance_page(p_chain, &p_chain->p_prod_elem, 310 p_prod_idx, p_prod_page_idx); 311 } 312 p_chain->u.chain16.prod_idx++; 313 } else { 314 if ((p_chain->u.chain32.prod_idx & 315 p_chain->elem_per_page_mask) == p_chain->next_page_mask) { 316 p_prod_idx = &p_chain->u.chain32.prod_idx; 317 p_prod_page_idx = &p_chain->pbl.u.pbl32.prod_page_idx; 318 qed_chain_advance_page(p_chain, &p_chain->p_prod_elem, 319 p_prod_idx, p_prod_page_idx); 320 } 321 p_chain->u.chain32.prod_idx++; 322 } 323 324 p_ret = p_chain->p_prod_elem; 325 p_chain->p_prod_elem = (void *)(((u8 *)p_chain->p_prod_elem) + 326 p_chain->elem_size); 327 328 return p_ret; 329 } 330 331 /** 332 * @brief qed_chain_get_capacity - 333 * 334 * Get the maximum number of BDs in chain 335 * 336 * @param p_chain 337 * @param num 338 * 339 * @return number of unusable BDs 340 */ 341 static inline u32 qed_chain_get_capacity(struct qed_chain *p_chain) 342 { 343 return p_chain->capacity; 344 } 345 346 /** 347 * @brief qed_chain_recycle_consumed - 348 * 349 * Returns an element which was previously consumed; 350 * Increments producers so they could be written to FW. 351 * 352 * @param p_chain 353 */ 354 static inline void qed_chain_recycle_consumed(struct qed_chain *p_chain) 355 { 356 test_and_skip(p_chain, prod_idx); 357 if (is_chain_u16(p_chain)) 358 p_chain->u.chain16.prod_idx++; 359 else 360 p_chain->u.chain32.prod_idx++; 361 } 362 363 /** 364 * @brief qed_chain_consume - 365 * 366 * A Chain in which the driver utilizes data written by a different source 367 * (i.e., FW) should use this to access passed buffers. 368 * 369 * @param p_chain 370 * 371 * @return void*, a pointer to the next buffer written 372 */ 373 static inline void *qed_chain_consume(struct qed_chain *p_chain) 374 { 375 void *p_ret = NULL, *p_cons_idx, *p_cons_page_idx; 376 377 if (is_chain_u16(p_chain)) { 378 if ((p_chain->u.chain16.cons_idx & 379 p_chain->elem_per_page_mask) == p_chain->next_page_mask) { 380 p_cons_idx = &p_chain->u.chain16.cons_idx; 381 p_cons_page_idx = &p_chain->pbl.u.pbl16.cons_page_idx; 382 qed_chain_advance_page(p_chain, &p_chain->p_cons_elem, 383 p_cons_idx, p_cons_page_idx); 384 } 385 p_chain->u.chain16.cons_idx++; 386 } else { 387 if ((p_chain->u.chain32.cons_idx & 388 p_chain->elem_per_page_mask) == p_chain->next_page_mask) { 389 p_cons_idx = &p_chain->u.chain32.cons_idx; 390 p_cons_page_idx = &p_chain->pbl.u.pbl32.cons_page_idx; 391 qed_chain_advance_page(p_chain, &p_chain->p_cons_elem, 392 p_cons_idx, p_cons_page_idx); 393 } 394 p_chain->u.chain32.cons_idx++; 395 } 396 397 p_ret = p_chain->p_cons_elem; 398 p_chain->p_cons_elem = (void *)(((u8 *)p_chain->p_cons_elem) + 399 p_chain->elem_size); 400 401 return p_ret; 402 } 403 404 /** 405 * @brief qed_chain_reset - Resets the chain to its start state 406 * 407 * @param p_chain pointer to a previously allocted chain 408 */ 409 static inline void qed_chain_reset(struct qed_chain *p_chain) 410 { 411 u32 i; 412 413 if (is_chain_u16(p_chain)) { 414 p_chain->u.chain16.prod_idx = 0; 415 p_chain->u.chain16.cons_idx = 0; 416 } else { 417 p_chain->u.chain32.prod_idx = 0; 418 p_chain->u.chain32.cons_idx = 0; 419 } 420 p_chain->p_cons_elem = p_chain->p_virt_addr; 421 p_chain->p_prod_elem = p_chain->p_virt_addr; 422 423 if (p_chain->mode == QED_CHAIN_MODE_PBL) { 424 /* Use (page_cnt - 1) as a reset value for the prod/cons page's 425 * indices, to avoid unnecessary page advancing on the first 426 * call to qed_chain_produce/consume. Instead, the indices 427 * will be advanced to page_cnt and then will be wrapped to 0. 428 */ 429 u32 reset_val = p_chain->page_cnt - 1; 430 431 if (is_chain_u16(p_chain)) { 432 p_chain->pbl.u.pbl16.prod_page_idx = (u16)reset_val; 433 p_chain->pbl.u.pbl16.cons_page_idx = (u16)reset_val; 434 } else { 435 p_chain->pbl.u.pbl32.prod_page_idx = reset_val; 436 p_chain->pbl.u.pbl32.cons_page_idx = reset_val; 437 } 438 } 439 440 switch (p_chain->intended_use) { 441 case QED_CHAIN_USE_TO_CONSUME_PRODUCE: 442 case QED_CHAIN_USE_TO_PRODUCE: 443 /* Do nothing */ 444 break; 445 446 case QED_CHAIN_USE_TO_CONSUME: 447 /* produce empty elements */ 448 for (i = 0; i < p_chain->capacity; i++) 449 qed_chain_recycle_consumed(p_chain); 450 break; 451 } 452 } 453 454 /** 455 * @brief qed_chain_init - Initalizes a basic chain struct 456 * 457 * @param p_chain 458 * @param p_virt_addr 459 * @param p_phys_addr physical address of allocated buffer's beginning 460 * @param page_cnt number of pages in the allocated buffer 461 * @param elem_size size of each element in the chain 462 * @param intended_use 463 * @param mode 464 */ 465 static inline void qed_chain_init_params(struct qed_chain *p_chain, 466 u32 page_cnt, 467 u8 elem_size, 468 enum qed_chain_use_mode intended_use, 469 enum qed_chain_mode mode, 470 enum qed_chain_cnt_type cnt_type) 471 { 472 /* chain fixed parameters */ 473 p_chain->p_virt_addr = NULL; 474 p_chain->p_phys_addr = 0; 475 p_chain->elem_size = elem_size; 476 p_chain->intended_use = intended_use; 477 p_chain->mode = mode; 478 p_chain->cnt_type = cnt_type; 479 480 p_chain->elem_per_page = ELEMS_PER_PAGE(elem_size); 481 p_chain->usable_per_page = USABLE_ELEMS_PER_PAGE(elem_size, mode); 482 p_chain->elem_per_page_mask = p_chain->elem_per_page - 1; 483 p_chain->elem_unusable = UNUSABLE_ELEMS_PER_PAGE(elem_size, mode); 484 p_chain->next_page_mask = (p_chain->usable_per_page & 485 p_chain->elem_per_page_mask); 486 487 p_chain->page_cnt = page_cnt; 488 p_chain->capacity = p_chain->usable_per_page * page_cnt; 489 p_chain->size = p_chain->elem_per_page * page_cnt; 490 491 p_chain->pbl.p_phys_table = 0; 492 p_chain->pbl.p_virt_table = NULL; 493 p_chain->pbl.pp_virt_addr_tbl = NULL; 494 } 495 496 /** 497 * @brief qed_chain_init_mem - 498 * 499 * Initalizes a basic chain struct with its chain buffers 500 * 501 * @param p_chain 502 * @param p_virt_addr virtual address of allocated buffer's beginning 503 * @param p_phys_addr physical address of allocated buffer's beginning 504 * 505 */ 506 static inline void qed_chain_init_mem(struct qed_chain *p_chain, 507 void *p_virt_addr, dma_addr_t p_phys_addr) 508 { 509 p_chain->p_virt_addr = p_virt_addr; 510 p_chain->p_phys_addr = p_phys_addr; 511 } 512 513 /** 514 * @brief qed_chain_init_pbl_mem - 515 * 516 * Initalizes a basic chain struct with its pbl buffers 517 * 518 * @param p_chain 519 * @param p_virt_pbl pointer to a pre allocated side table which will hold 520 * virtual page addresses. 521 * @param p_phys_pbl pointer to a pre-allocated side table which will hold 522 * physical page addresses. 523 * @param pp_virt_addr_tbl 524 * pointer to a pre-allocated side table which will hold 525 * the virtual addresses of the chain pages. 526 * 527 */ 528 static inline void qed_chain_init_pbl_mem(struct qed_chain *p_chain, 529 void *p_virt_pbl, 530 dma_addr_t p_phys_pbl, 531 void **pp_virt_addr_tbl) 532 { 533 p_chain->pbl.p_phys_table = p_phys_pbl; 534 p_chain->pbl.p_virt_table = p_virt_pbl; 535 p_chain->pbl.pp_virt_addr_tbl = pp_virt_addr_tbl; 536 } 537 538 /** 539 * @brief qed_chain_init_next_ptr_elem - 540 * 541 * Initalizes a next pointer element 542 * 543 * @param p_chain 544 * @param p_virt_curr virtual address of a chain page of which the next 545 * pointer element is initialized 546 * @param p_virt_next virtual address of the next chain page 547 * @param p_phys_next physical address of the next chain page 548 * 549 */ 550 static inline void 551 qed_chain_init_next_ptr_elem(struct qed_chain *p_chain, 552 void *p_virt_curr, 553 void *p_virt_next, dma_addr_t p_phys_next) 554 { 555 struct qed_chain_next *p_next; 556 u32 size; 557 558 size = p_chain->elem_size * p_chain->usable_per_page; 559 p_next = (struct qed_chain_next *)((u8 *)p_virt_curr + size); 560 561 DMA_REGPAIR_LE(p_next->next_phys, p_phys_next); 562 563 p_next->next_virt = p_virt_next; 564 } 565 566 /** 567 * @brief qed_chain_get_last_elem - 568 * 569 * Returns a pointer to the last element of the chain 570 * 571 * @param p_chain 572 * 573 * @return void* 574 */ 575 static inline void *qed_chain_get_last_elem(struct qed_chain *p_chain) 576 { 577 struct qed_chain_next *p_next = NULL; 578 void *p_virt_addr = NULL; 579 u32 size, last_page_idx; 580 581 if (!p_chain->p_virt_addr) 582 goto out; 583 584 switch (p_chain->mode) { 585 case QED_CHAIN_MODE_NEXT_PTR: 586 size = p_chain->elem_size * p_chain->usable_per_page; 587 p_virt_addr = p_chain->p_virt_addr; 588 p_next = (struct qed_chain_next *)((u8 *)p_virt_addr + size); 589 while (p_next->next_virt != p_chain->p_virt_addr) { 590 p_virt_addr = p_next->next_virt; 591 p_next = (struct qed_chain_next *)((u8 *)p_virt_addr + 592 size); 593 } 594 break; 595 case QED_CHAIN_MODE_SINGLE: 596 p_virt_addr = p_chain->p_virt_addr; 597 break; 598 case QED_CHAIN_MODE_PBL: 599 last_page_idx = p_chain->page_cnt - 1; 600 p_virt_addr = p_chain->pbl.pp_virt_addr_tbl[last_page_idx]; 601 break; 602 } 603 /* p_virt_addr points at this stage to the last page of the chain */ 604 size = p_chain->elem_size * (p_chain->usable_per_page - 1); 605 p_virt_addr = (u8 *)p_virt_addr + size; 606 out: 607 return p_virt_addr; 608 } 609 610 /** 611 * @brief qed_chain_set_prod - sets the prod to the given value 612 * 613 * @param prod_idx 614 * @param p_prod_elem 615 */ 616 static inline void qed_chain_set_prod(struct qed_chain *p_chain, 617 u32 prod_idx, void *p_prod_elem) 618 { 619 if (is_chain_u16(p_chain)) 620 p_chain->u.chain16.prod_idx = (u16) prod_idx; 621 else 622 p_chain->u.chain32.prod_idx = prod_idx; 623 p_chain->p_prod_elem = p_prod_elem; 624 } 625 626 /** 627 * @brief qed_chain_pbl_zero_mem - set chain memory to 0 628 * 629 * @param p_chain 630 */ 631 static inline void qed_chain_pbl_zero_mem(struct qed_chain *p_chain) 632 { 633 u32 i, page_cnt; 634 635 if (p_chain->mode != QED_CHAIN_MODE_PBL) 636 return; 637 638 page_cnt = qed_chain_get_page_cnt(p_chain); 639 640 for (i = 0; i < page_cnt; i++) 641 memset(p_chain->pbl.pp_virt_addr_tbl[i], 0, 642 QED_CHAIN_PAGE_SIZE); 643 } 644 645 #endif 646