1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2001-2018 3 */ 4 5 #include "i40e_status.h" 6 #include "i40e_type.h" 7 #include "i40e_register.h" 8 #include "i40e_adminq.h" 9 #include "i40e_prototype.h" 10 11 /** 12 * i40e_adminq_init_regs - Initialize AdminQ registers 13 * @hw: pointer to the hardware structure 14 * 15 * This assumes the alloc_asq and alloc_arq functions have already been called 16 **/ 17 STATIC void i40e_adminq_init_regs(struct i40e_hw *hw) 18 { 19 /* set head and tail registers in our local struct */ 20 if (i40e_is_vf(hw)) { 21 hw->aq.asq.tail = I40E_VF_ATQT1; 22 hw->aq.asq.head = I40E_VF_ATQH1; 23 hw->aq.asq.len = I40E_VF_ATQLEN1; 24 hw->aq.asq.bal = I40E_VF_ATQBAL1; 25 hw->aq.asq.bah = I40E_VF_ATQBAH1; 26 hw->aq.arq.tail = I40E_VF_ARQT1; 27 hw->aq.arq.head = I40E_VF_ARQH1; 28 hw->aq.arq.len = I40E_VF_ARQLEN1; 29 hw->aq.arq.bal = I40E_VF_ARQBAL1; 30 hw->aq.arq.bah = I40E_VF_ARQBAH1; 31 #ifdef PF_DRIVER 32 } else { 33 hw->aq.asq.tail = I40E_PF_ATQT; 34 hw->aq.asq.head = I40E_PF_ATQH; 35 hw->aq.asq.len = I40E_PF_ATQLEN; 36 hw->aq.asq.bal = I40E_PF_ATQBAL; 37 hw->aq.asq.bah = I40E_PF_ATQBAH; 38 hw->aq.arq.tail = I40E_PF_ARQT; 39 hw->aq.arq.head = I40E_PF_ARQH; 40 hw->aq.arq.len = I40E_PF_ARQLEN; 41 hw->aq.arq.bal = I40E_PF_ARQBAL; 42 hw->aq.arq.bah = I40E_PF_ARQBAH; 43 #endif 44 } 45 } 46 47 /** 48 * i40e_alloc_adminq_asq_ring - Allocate Admin Queue send rings 49 * @hw: pointer to the hardware structure 50 **/ 51 enum i40e_status_code i40e_alloc_adminq_asq_ring(struct i40e_hw *hw) 52 { 53 enum i40e_status_code ret_code; 54 55 ret_code = i40e_allocate_dma_mem(hw, &hw->aq.asq.desc_buf, 56 i40e_mem_atq_ring, 57 (hw->aq.num_asq_entries * 58 sizeof(struct i40e_aq_desc)), 59 I40E_ADMINQ_DESC_ALIGNMENT); 60 if (ret_code) 61 return ret_code; 62 63 ret_code = i40e_allocate_virt_mem(hw, &hw->aq.asq.cmd_buf, 64 (hw->aq.num_asq_entries * 65 sizeof(struct i40e_asq_cmd_details))); 66 if (ret_code) { 67 i40e_free_dma_mem(hw, &hw->aq.asq.desc_buf); 68 return ret_code; 69 } 70 71 return ret_code; 72 } 73 74 /** 75 * i40e_alloc_adminq_arq_ring - Allocate Admin Queue receive rings 76 * @hw: pointer to the hardware structure 77 **/ 78 enum i40e_status_code i40e_alloc_adminq_arq_ring(struct i40e_hw *hw) 79 { 80 enum i40e_status_code ret_code; 81 82 ret_code = i40e_allocate_dma_mem(hw, &hw->aq.arq.desc_buf, 83 i40e_mem_arq_ring, 84 (hw->aq.num_arq_entries * 85 sizeof(struct i40e_aq_desc)), 86 I40E_ADMINQ_DESC_ALIGNMENT); 87 88 return ret_code; 89 } 90 91 /** 92 * i40e_free_adminq_asq - Free Admin Queue send rings 93 * @hw: pointer to the hardware structure 94 * 95 * This assumes the posted send buffers have already been cleaned 96 * and de-allocated 97 **/ 98 void i40e_free_adminq_asq(struct i40e_hw *hw) 99 { 100 i40e_free_virt_mem(hw, &hw->aq.asq.cmd_buf); 101 i40e_free_dma_mem(hw, &hw->aq.asq.desc_buf); 102 } 103 104 /** 105 * i40e_free_adminq_arq - Free Admin Queue receive rings 106 * @hw: pointer to the hardware structure 107 * 108 * This assumes the posted receive buffers have already been cleaned 109 * and de-allocated 110 **/ 111 void i40e_free_adminq_arq(struct i40e_hw *hw) 112 { 113 i40e_free_dma_mem(hw, &hw->aq.arq.desc_buf); 114 } 115 116 /** 117 * i40e_alloc_arq_bufs - Allocate pre-posted buffers for the receive queue 118 * @hw: pointer to the hardware structure 119 **/ 120 STATIC enum i40e_status_code i40e_alloc_arq_bufs(struct i40e_hw *hw) 121 { 122 enum i40e_status_code ret_code; 123 struct i40e_aq_desc *desc; 124 struct i40e_dma_mem *bi; 125 int i; 126 127 /* We'll be allocating the buffer info memory first, then we can 128 * allocate the mapped buffers for the event processing 129 */ 130 131 /* buffer_info structures do not need alignment */ 132 ret_code = i40e_allocate_virt_mem(hw, &hw->aq.arq.dma_head, 133 (hw->aq.num_arq_entries * sizeof(struct i40e_dma_mem))); 134 if (ret_code) 135 goto alloc_arq_bufs; 136 hw->aq.arq.r.arq_bi = (struct i40e_dma_mem *)hw->aq.arq.dma_head.va; 137 138 /* allocate the mapped buffers */ 139 for (i = 0; i < hw->aq.num_arq_entries; i++) { 140 bi = &hw->aq.arq.r.arq_bi[i]; 141 ret_code = i40e_allocate_dma_mem(hw, bi, 142 i40e_mem_arq_buf, 143 hw->aq.arq_buf_size, 144 I40E_ADMINQ_DESC_ALIGNMENT); 145 if (ret_code) 146 goto unwind_alloc_arq_bufs; 147 148 /* now configure the descriptors for use */ 149 desc = I40E_ADMINQ_DESC(hw->aq.arq, i); 150 151 desc->flags = CPU_TO_LE16(I40E_AQ_FLAG_BUF); 152 if (hw->aq.arq_buf_size > I40E_AQ_LARGE_BUF) 153 desc->flags |= CPU_TO_LE16(I40E_AQ_FLAG_LB); 154 desc->opcode = 0; 155 /* This is in accordance with Admin queue design, there is no 156 * register for buffer size configuration 157 */ 158 desc->datalen = CPU_TO_LE16((u16)bi->size); 159 desc->retval = 0; 160 desc->cookie_high = 0; 161 desc->cookie_low = 0; 162 desc->params.external.addr_high = 163 CPU_TO_LE32(I40E_HI_DWORD(bi->pa)); 164 desc->params.external.addr_low = 165 CPU_TO_LE32(I40E_LO_DWORD(bi->pa)); 166 desc->params.external.param0 = 0; 167 desc->params.external.param1 = 0; 168 } 169 170 alloc_arq_bufs: 171 return ret_code; 172 173 unwind_alloc_arq_bufs: 174 /* don't try to free the one that failed... */ 175 i--; 176 for (; i >= 0; i--) 177 i40e_free_dma_mem(hw, &hw->aq.arq.r.arq_bi[i]); 178 i40e_free_virt_mem(hw, &hw->aq.arq.dma_head); 179 180 return ret_code; 181 } 182 183 /** 184 * i40e_alloc_asq_bufs - Allocate empty buffer structs for the send queue 185 * @hw: pointer to the hardware structure 186 **/ 187 STATIC enum i40e_status_code i40e_alloc_asq_bufs(struct i40e_hw *hw) 188 { 189 enum i40e_status_code ret_code; 190 struct i40e_dma_mem *bi; 191 int i; 192 193 /* No mapped memory needed yet, just the buffer info structures */ 194 ret_code = i40e_allocate_virt_mem(hw, &hw->aq.asq.dma_head, 195 (hw->aq.num_asq_entries * sizeof(struct i40e_dma_mem))); 196 if (ret_code) 197 goto alloc_asq_bufs; 198 hw->aq.asq.r.asq_bi = (struct i40e_dma_mem *)hw->aq.asq.dma_head.va; 199 200 /* allocate the mapped buffers */ 201 for (i = 0; i < hw->aq.num_asq_entries; i++) { 202 bi = &hw->aq.asq.r.asq_bi[i]; 203 ret_code = i40e_allocate_dma_mem(hw, bi, 204 i40e_mem_asq_buf, 205 hw->aq.asq_buf_size, 206 I40E_ADMINQ_DESC_ALIGNMENT); 207 if (ret_code) 208 goto unwind_alloc_asq_bufs; 209 } 210 alloc_asq_bufs: 211 return ret_code; 212 213 unwind_alloc_asq_bufs: 214 /* don't try to free the one that failed... */ 215 i--; 216 for (; i >= 0; i--) 217 i40e_free_dma_mem(hw, &hw->aq.asq.r.asq_bi[i]); 218 i40e_free_virt_mem(hw, &hw->aq.asq.dma_head); 219 220 return ret_code; 221 } 222 223 /** 224 * i40e_free_arq_bufs - Free receive queue buffer info elements 225 * @hw: pointer to the hardware structure 226 **/ 227 STATIC void i40e_free_arq_bufs(struct i40e_hw *hw) 228 { 229 int i; 230 231 /* free descriptors */ 232 for (i = 0; i < hw->aq.num_arq_entries; i++) 233 i40e_free_dma_mem(hw, &hw->aq.arq.r.arq_bi[i]); 234 235 /* free the descriptor memory */ 236 i40e_free_dma_mem(hw, &hw->aq.arq.desc_buf); 237 238 /* free the dma header */ 239 i40e_free_virt_mem(hw, &hw->aq.arq.dma_head); 240 } 241 242 /** 243 * i40e_free_asq_bufs - Free send queue buffer info elements 244 * @hw: pointer to the hardware structure 245 **/ 246 STATIC void i40e_free_asq_bufs(struct i40e_hw *hw) 247 { 248 int i; 249 250 /* only unmap if the address is non-NULL */ 251 for (i = 0; i < hw->aq.num_asq_entries; i++) 252 if (hw->aq.asq.r.asq_bi[i].pa) 253 i40e_free_dma_mem(hw, &hw->aq.asq.r.asq_bi[i]); 254 255 /* free the buffer info list */ 256 i40e_free_virt_mem(hw, &hw->aq.asq.cmd_buf); 257 258 /* free the descriptor memory */ 259 i40e_free_dma_mem(hw, &hw->aq.asq.desc_buf); 260 261 /* free the dma header */ 262 i40e_free_virt_mem(hw, &hw->aq.asq.dma_head); 263 } 264 265 /** 266 * i40e_config_asq_regs - configure ASQ registers 267 * @hw: pointer to the hardware structure 268 * 269 * Configure base address and length registers for the transmit queue 270 **/ 271 STATIC enum i40e_status_code i40e_config_asq_regs(struct i40e_hw *hw) 272 { 273 enum i40e_status_code ret_code = I40E_SUCCESS; 274 u32 reg = 0; 275 276 /* Clear Head and Tail */ 277 wr32(hw, hw->aq.asq.head, 0); 278 wr32(hw, hw->aq.asq.tail, 0); 279 280 /* set starting point */ 281 #ifdef PF_DRIVER 282 #ifdef INTEGRATED_VF 283 if (!i40e_is_vf(hw)) 284 wr32(hw, hw->aq.asq.len, (hw->aq.num_asq_entries | 285 I40E_PF_ATQLEN_ATQENABLE_MASK)); 286 #else 287 wr32(hw, hw->aq.asq.len, (hw->aq.num_asq_entries | 288 I40E_PF_ATQLEN_ATQENABLE_MASK)); 289 #endif /* INTEGRATED_VF */ 290 #endif /* PF_DRIVER */ 291 #ifdef VF_DRIVER 292 #ifdef INTEGRATED_VF 293 if (i40e_is_vf(hw)) 294 wr32(hw, hw->aq.asq.len, (hw->aq.num_asq_entries | 295 I40E_VF_ATQLEN1_ATQENABLE_MASK)); 296 #else 297 wr32(hw, hw->aq.asq.len, (hw->aq.num_asq_entries | 298 I40E_VF_ATQLEN1_ATQENABLE_MASK)); 299 #endif /* INTEGRATED_VF */ 300 #endif /* VF_DRIVER */ 301 wr32(hw, hw->aq.asq.bal, I40E_LO_DWORD(hw->aq.asq.desc_buf.pa)); 302 wr32(hw, hw->aq.asq.bah, I40E_HI_DWORD(hw->aq.asq.desc_buf.pa)); 303 304 /* Check one register to verify that config was applied */ 305 reg = rd32(hw, hw->aq.asq.bal); 306 if (reg != I40E_LO_DWORD(hw->aq.asq.desc_buf.pa)) 307 ret_code = I40E_ERR_ADMIN_QUEUE_ERROR; 308 309 return ret_code; 310 } 311 312 /** 313 * i40e_config_arq_regs - ARQ register configuration 314 * @hw: pointer to the hardware structure 315 * 316 * Configure base address and length registers for the receive (event queue) 317 **/ 318 STATIC enum i40e_status_code i40e_config_arq_regs(struct i40e_hw *hw) 319 { 320 enum i40e_status_code ret_code = I40E_SUCCESS; 321 u32 reg = 0; 322 323 /* Clear Head and Tail */ 324 wr32(hw, hw->aq.arq.head, 0); 325 wr32(hw, hw->aq.arq.tail, 0); 326 327 /* set starting point */ 328 #ifdef PF_DRIVER 329 #ifdef INTEGRATED_VF 330 if (!i40e_is_vf(hw)) 331 wr32(hw, hw->aq.arq.len, (hw->aq.num_arq_entries | 332 I40E_PF_ARQLEN_ARQENABLE_MASK)); 333 #else 334 wr32(hw, hw->aq.arq.len, (hw->aq.num_arq_entries | 335 I40E_PF_ARQLEN_ARQENABLE_MASK)); 336 #endif /* INTEGRATED_VF */ 337 #endif /* PF_DRIVER */ 338 #ifdef VF_DRIVER 339 #ifdef INTEGRATED_VF 340 if (i40e_is_vf(hw)) 341 wr32(hw, hw->aq.arq.len, (hw->aq.num_arq_entries | 342 I40E_VF_ARQLEN1_ARQENABLE_MASK)); 343 #else 344 wr32(hw, hw->aq.arq.len, (hw->aq.num_arq_entries | 345 I40E_VF_ARQLEN1_ARQENABLE_MASK)); 346 #endif /* INTEGRATED_VF */ 347 #endif /* VF_DRIVER */ 348 wr32(hw, hw->aq.arq.bal, I40E_LO_DWORD(hw->aq.arq.desc_buf.pa)); 349 wr32(hw, hw->aq.arq.bah, I40E_HI_DWORD(hw->aq.arq.desc_buf.pa)); 350 351 /* Update tail in the HW to post pre-allocated buffers */ 352 wr32(hw, hw->aq.arq.tail, hw->aq.num_arq_entries - 1); 353 354 /* Check one register to verify that config was applied */ 355 reg = rd32(hw, hw->aq.arq.bal); 356 if (reg != I40E_LO_DWORD(hw->aq.arq.desc_buf.pa)) 357 ret_code = I40E_ERR_ADMIN_QUEUE_ERROR; 358 359 return ret_code; 360 } 361 362 /** 363 * i40e_init_asq - main initialization routine for ASQ 364 * @hw: pointer to the hardware structure 365 * 366 * This is the main initialization routine for the Admin Send Queue 367 * Prior to calling this function, drivers *MUST* set the following fields 368 * in the hw->aq structure: 369 * - hw->aq.num_asq_entries 370 * - hw->aq.arq_buf_size 371 * 372 * Do *NOT* hold the lock when calling this as the memory allocation routines 373 * called are not going to be atomic context safe 374 **/ 375 enum i40e_status_code i40e_init_asq(struct i40e_hw *hw) 376 { 377 enum i40e_status_code ret_code = I40E_SUCCESS; 378 379 if (hw->aq.asq.count > 0) { 380 /* queue already initialized */ 381 ret_code = I40E_ERR_NOT_READY; 382 goto init_adminq_exit; 383 } 384 385 /* verify input for valid configuration */ 386 if ((hw->aq.num_asq_entries == 0) || 387 (hw->aq.asq_buf_size == 0)) { 388 ret_code = I40E_ERR_CONFIG; 389 goto init_adminq_exit; 390 } 391 392 hw->aq.asq.next_to_use = 0; 393 hw->aq.asq.next_to_clean = 0; 394 395 /* allocate the ring memory */ 396 ret_code = i40e_alloc_adminq_asq_ring(hw); 397 if (ret_code != I40E_SUCCESS) 398 goto init_adminq_exit; 399 400 /* allocate buffers in the rings */ 401 ret_code = i40e_alloc_asq_bufs(hw); 402 if (ret_code != I40E_SUCCESS) 403 goto init_adminq_free_rings; 404 405 /* initialize base registers */ 406 ret_code = i40e_config_asq_regs(hw); 407 if (ret_code != I40E_SUCCESS) 408 goto init_config_regs; 409 410 /* success! */ 411 hw->aq.asq.count = hw->aq.num_asq_entries; 412 goto init_adminq_exit; 413 414 init_adminq_free_rings: 415 i40e_free_adminq_asq(hw); 416 return ret_code; 417 418 init_config_regs: 419 i40e_free_asq_bufs(hw); 420 421 init_adminq_exit: 422 return ret_code; 423 } 424 425 /** 426 * i40e_init_arq - initialize ARQ 427 * @hw: pointer to the hardware structure 428 * 429 * The main initialization routine for the Admin Receive (Event) Queue. 430 * Prior to calling this function, drivers *MUST* set the following fields 431 * in the hw->aq structure: 432 * - hw->aq.num_asq_entries 433 * - hw->aq.arq_buf_size 434 * 435 * Do *NOT* hold the lock when calling this as the memory allocation routines 436 * called are not going to be atomic context safe 437 **/ 438 enum i40e_status_code i40e_init_arq(struct i40e_hw *hw) 439 { 440 enum i40e_status_code ret_code = I40E_SUCCESS; 441 442 if (hw->aq.arq.count > 0) { 443 /* queue already initialized */ 444 ret_code = I40E_ERR_NOT_READY; 445 goto init_adminq_exit; 446 } 447 448 /* verify input for valid configuration */ 449 if ((hw->aq.num_arq_entries == 0) || 450 (hw->aq.arq_buf_size == 0)) { 451 ret_code = I40E_ERR_CONFIG; 452 goto init_adminq_exit; 453 } 454 455 hw->aq.arq.next_to_use = 0; 456 hw->aq.arq.next_to_clean = 0; 457 458 /* allocate the ring memory */ 459 ret_code = i40e_alloc_adminq_arq_ring(hw); 460 if (ret_code != I40E_SUCCESS) 461 goto init_adminq_exit; 462 463 /* allocate buffers in the rings */ 464 ret_code = i40e_alloc_arq_bufs(hw); 465 if (ret_code != I40E_SUCCESS) 466 goto init_adminq_free_rings; 467 468 /* initialize base registers */ 469 ret_code = i40e_config_arq_regs(hw); 470 if (ret_code != I40E_SUCCESS) 471 goto init_adminq_free_rings; 472 473 /* success! */ 474 hw->aq.arq.count = hw->aq.num_arq_entries; 475 goto init_adminq_exit; 476 477 init_adminq_free_rings: 478 i40e_free_adminq_arq(hw); 479 480 init_adminq_exit: 481 return ret_code; 482 } 483 484 /** 485 * i40e_shutdown_asq - shutdown the ASQ 486 * @hw: pointer to the hardware structure 487 * 488 * The main shutdown routine for the Admin Send Queue 489 **/ 490 enum i40e_status_code i40e_shutdown_asq(struct i40e_hw *hw) 491 { 492 enum i40e_status_code ret_code = I40E_SUCCESS; 493 494 i40e_acquire_spinlock(&hw->aq.asq_spinlock); 495 496 if (hw->aq.asq.count == 0) { 497 ret_code = I40E_ERR_NOT_READY; 498 goto shutdown_asq_out; 499 } 500 501 /* Stop firmware AdminQ processing */ 502 wr32(hw, hw->aq.asq.head, 0); 503 wr32(hw, hw->aq.asq.tail, 0); 504 wr32(hw, hw->aq.asq.len, 0); 505 wr32(hw, hw->aq.asq.bal, 0); 506 wr32(hw, hw->aq.asq.bah, 0); 507 508 hw->aq.asq.count = 0; /* to indicate uninitialized queue */ 509 510 /* free ring buffers */ 511 i40e_free_asq_bufs(hw); 512 513 shutdown_asq_out: 514 i40e_release_spinlock(&hw->aq.asq_spinlock); 515 return ret_code; 516 } 517 518 /** 519 * i40e_shutdown_arq - shutdown ARQ 520 * @hw: pointer to the hardware structure 521 * 522 * The main shutdown routine for the Admin Receive Queue 523 **/ 524 enum i40e_status_code i40e_shutdown_arq(struct i40e_hw *hw) 525 { 526 enum i40e_status_code ret_code = I40E_SUCCESS; 527 528 i40e_acquire_spinlock(&hw->aq.arq_spinlock); 529 530 if (hw->aq.arq.count == 0) { 531 ret_code = I40E_ERR_NOT_READY; 532 goto shutdown_arq_out; 533 } 534 535 /* Stop firmware AdminQ processing */ 536 wr32(hw, hw->aq.arq.head, 0); 537 wr32(hw, hw->aq.arq.tail, 0); 538 wr32(hw, hw->aq.arq.len, 0); 539 wr32(hw, hw->aq.arq.bal, 0); 540 wr32(hw, hw->aq.arq.bah, 0); 541 542 hw->aq.arq.count = 0; /* to indicate uninitialized queue */ 543 544 /* free ring buffers */ 545 i40e_free_arq_bufs(hw); 546 547 shutdown_arq_out: 548 i40e_release_spinlock(&hw->aq.arq_spinlock); 549 return ret_code; 550 } 551 #ifdef PF_DRIVER 552 553 /** 554 * i40e_resume_aq - resume AQ processing from 0 555 * @hw: pointer to the hardware structure 556 **/ 557 STATIC void i40e_resume_aq(struct i40e_hw *hw) 558 { 559 /* Registers are reset after PF reset */ 560 hw->aq.asq.next_to_use = 0; 561 hw->aq.asq.next_to_clean = 0; 562 563 i40e_config_asq_regs(hw); 564 565 hw->aq.arq.next_to_use = 0; 566 hw->aq.arq.next_to_clean = 0; 567 568 i40e_config_arq_regs(hw); 569 } 570 #endif /* PF_DRIVER */ 571 572 /** 573 * i40e_init_adminq - main initialization routine for Admin Queue 574 * @hw: pointer to the hardware structure 575 * 576 * Prior to calling this function, drivers *MUST* set the following fields 577 * in the hw->aq structure: 578 * - hw->aq.num_asq_entries 579 * - hw->aq.num_arq_entries 580 * - hw->aq.arq_buf_size 581 * - hw->aq.asq_buf_size 582 **/ 583 enum i40e_status_code i40e_init_adminq(struct i40e_hw *hw) 584 { 585 #ifdef PF_DRIVER 586 u16 cfg_ptr, oem_hi, oem_lo; 587 u16 eetrack_lo, eetrack_hi; 588 #endif 589 enum i40e_status_code ret_code; 590 #ifdef PF_DRIVER 591 int retry = 0; 592 #endif 593 594 /* verify input for valid configuration */ 595 if ((hw->aq.num_arq_entries == 0) || 596 (hw->aq.num_asq_entries == 0) || 597 (hw->aq.arq_buf_size == 0) || 598 (hw->aq.asq_buf_size == 0)) { 599 ret_code = I40E_ERR_CONFIG; 600 goto init_adminq_exit; 601 } 602 i40e_init_spinlock(&hw->aq.asq_spinlock); 603 i40e_init_spinlock(&hw->aq.arq_spinlock); 604 605 /* Set up register offsets */ 606 i40e_adminq_init_regs(hw); 607 608 /* setup ASQ command write back timeout */ 609 hw->aq.asq_cmd_timeout = I40E_ASQ_CMD_TIMEOUT; 610 611 /* allocate the ASQ */ 612 ret_code = i40e_init_asq(hw); 613 if (ret_code != I40E_SUCCESS) 614 goto init_adminq_destroy_spinlocks; 615 616 /* allocate the ARQ */ 617 ret_code = i40e_init_arq(hw); 618 if (ret_code != I40E_SUCCESS) 619 goto init_adminq_free_asq; 620 621 #ifdef PF_DRIVER 622 #ifdef INTEGRATED_VF 623 /* VF has no need of firmware */ 624 if (i40e_is_vf(hw)) 625 goto init_adminq_exit; 626 #endif 627 /* There are some cases where the firmware may not be quite ready 628 * for AdminQ operations, so we retry the AdminQ setup a few times 629 * if we see timeouts in this first AQ call. 630 */ 631 do { 632 ret_code = i40e_aq_get_firmware_version(hw, 633 &hw->aq.fw_maj_ver, 634 &hw->aq.fw_min_ver, 635 &hw->aq.fw_build, 636 &hw->aq.api_maj_ver, 637 &hw->aq.api_min_ver, 638 NULL); 639 if (ret_code != I40E_ERR_ADMIN_QUEUE_TIMEOUT) 640 break; 641 retry++; 642 i40e_msec_delay(100); 643 i40e_resume_aq(hw); 644 } while (retry < 10); 645 if (ret_code != I40E_SUCCESS) 646 goto init_adminq_free_arq; 647 648 /* get the NVM version info */ 649 i40e_read_nvm_word(hw, I40E_SR_NVM_DEV_STARTER_VERSION, 650 &hw->nvm.version); 651 i40e_read_nvm_word(hw, I40E_SR_NVM_EETRACK_LO, &eetrack_lo); 652 i40e_read_nvm_word(hw, I40E_SR_NVM_EETRACK_HI, &eetrack_hi); 653 hw->nvm.eetrack = (eetrack_hi << 16) | eetrack_lo; 654 i40e_read_nvm_word(hw, I40E_SR_BOOT_CONFIG_PTR, &cfg_ptr); 655 i40e_read_nvm_word(hw, (cfg_ptr + I40E_NVM_OEM_VER_OFF), 656 &oem_hi); 657 i40e_read_nvm_word(hw, (cfg_ptr + (I40E_NVM_OEM_VER_OFF + 1)), 658 &oem_lo); 659 hw->nvm.oem_ver = ((u32)oem_hi << 16) | oem_lo; 660 661 /* The ability to RX (not drop) 802.1ad frames was added in API 1.7 */ 662 if ((hw->aq.api_maj_ver > 1) || 663 ((hw->aq.api_maj_ver == 1) && 664 (hw->aq.api_min_ver >= 7))) 665 hw->flags |= I40E_HW_FLAG_802_1AD_CAPABLE; 666 667 if (hw->mac.type == I40E_MAC_XL710 && 668 hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && 669 hw->aq.api_min_ver >= I40E_MINOR_VER_GET_LINK_INFO_XL710) { 670 hw->flags |= I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE; 671 hw->flags |= I40E_HW_FLAG_FW_LLDP_STOPPABLE; 672 } 673 if (hw->mac.type == I40E_MAC_X722 && 674 hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && 675 hw->aq.api_min_ver >= I40E_MINOR_VER_FW_LLDP_STOPPABLE_X722) { 676 hw->flags |= I40E_HW_FLAG_FW_LLDP_STOPPABLE; 677 } 678 679 /* Newer versions of firmware require lock when reading the NVM */ 680 if ((hw->aq.api_maj_ver > 1) || 681 ((hw->aq.api_maj_ver == 1) && 682 (hw->aq.api_min_ver >= 5))) 683 hw->flags |= I40E_HW_FLAG_NVM_READ_REQUIRES_LOCK; 684 685 if (hw->aq.api_maj_ver > I40E_FW_API_VERSION_MAJOR) { 686 ret_code = I40E_ERR_FIRMWARE_API_VERSION; 687 goto init_adminq_free_arq; 688 } 689 690 /* pre-emptive resource lock release */ 691 i40e_aq_release_resource(hw, I40E_NVM_RESOURCE_ID, 0, NULL); 692 hw->nvm_release_on_done = false; 693 hw->nvmupd_state = I40E_NVMUPD_STATE_INIT; 694 695 #endif /* PF_DRIVER */ 696 ret_code = I40E_SUCCESS; 697 698 /* success! */ 699 goto init_adminq_exit; 700 701 #ifdef PF_DRIVER 702 init_adminq_free_arq: 703 i40e_shutdown_arq(hw); 704 #endif 705 init_adminq_free_asq: 706 i40e_shutdown_asq(hw); 707 init_adminq_destroy_spinlocks: 708 i40e_destroy_spinlock(&hw->aq.asq_spinlock); 709 i40e_destroy_spinlock(&hw->aq.arq_spinlock); 710 711 init_adminq_exit: 712 return ret_code; 713 } 714 715 /** 716 * i40e_shutdown_adminq - shutdown routine for the Admin Queue 717 * @hw: pointer to the hardware structure 718 **/ 719 enum i40e_status_code i40e_shutdown_adminq(struct i40e_hw *hw) 720 { 721 enum i40e_status_code ret_code = I40E_SUCCESS; 722 723 if (i40e_check_asq_alive(hw)) 724 i40e_aq_queue_shutdown(hw, true); 725 726 i40e_shutdown_asq(hw); 727 i40e_shutdown_arq(hw); 728 i40e_destroy_spinlock(&hw->aq.asq_spinlock); 729 i40e_destroy_spinlock(&hw->aq.arq_spinlock); 730 731 if (hw->nvm_buff.va) 732 i40e_free_virt_mem(hw, &hw->nvm_buff); 733 734 return ret_code; 735 } 736 737 /** 738 * i40e_clean_asq - cleans Admin send queue 739 * @hw: pointer to the hardware structure 740 * 741 * returns the number of free desc 742 **/ 743 u16 i40e_clean_asq(struct i40e_hw *hw) 744 { 745 struct i40e_adminq_ring *asq = &(hw->aq.asq); 746 struct i40e_asq_cmd_details *details; 747 u16 ntc = asq->next_to_clean; 748 struct i40e_aq_desc desc_cb; 749 struct i40e_aq_desc *desc; 750 751 desc = I40E_ADMINQ_DESC(*asq, ntc); 752 details = I40E_ADMINQ_DETAILS(*asq, ntc); 753 while (rd32(hw, hw->aq.asq.head) != ntc) { 754 i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, 755 "ntc %d head %d.\n", ntc, rd32(hw, hw->aq.asq.head)); 756 757 if (details->callback) { 758 I40E_ADMINQ_CALLBACK cb_func = 759 (I40E_ADMINQ_CALLBACK)details->callback; 760 i40e_memcpy(&desc_cb, desc, sizeof(struct i40e_aq_desc), 761 I40E_DMA_TO_DMA); 762 cb_func(hw, &desc_cb); 763 } 764 i40e_memset(desc, 0, sizeof(*desc), I40E_DMA_MEM); 765 i40e_memset(details, 0, sizeof(*details), I40E_NONDMA_MEM); 766 ntc++; 767 if (ntc == asq->count) 768 ntc = 0; 769 desc = I40E_ADMINQ_DESC(*asq, ntc); 770 details = I40E_ADMINQ_DETAILS(*asq, ntc); 771 } 772 773 asq->next_to_clean = ntc; 774 775 return I40E_DESC_UNUSED(asq); 776 } 777 778 /** 779 * i40e_asq_done - check if FW has processed the Admin Send Queue 780 * @hw: pointer to the hw struct 781 * 782 * Returns true if the firmware has processed all descriptors on the 783 * admin send queue. Returns false if there are still requests pending. 784 **/ 785 #ifdef VF_DRIVER 786 bool i40e_asq_done(struct i40e_hw *hw) 787 #else 788 STATIC bool i40e_asq_done(struct i40e_hw *hw) 789 #endif 790 { 791 /* AQ designers suggest use of head for better 792 * timing reliability than DD bit 793 */ 794 return rd32(hw, hw->aq.asq.head) == hw->aq.asq.next_to_use; 795 796 } 797 798 /** 799 * i40e_asq_send_command - send command to Admin Queue 800 * @hw: pointer to the hw struct 801 * @desc: prefilled descriptor describing the command (non DMA mem) 802 * @buff: buffer to use for indirect commands 803 * @buff_size: size of buffer for indirect commands 804 * @cmd_details: pointer to command details structure 805 * 806 * This is the main send command driver routine for the Admin Queue send 807 * queue. It runs the queue, cleans the queue, etc 808 **/ 809 enum i40e_status_code i40e_asq_send_command(struct i40e_hw *hw, 810 struct i40e_aq_desc *desc, 811 void *buff, /* can be NULL */ 812 u16 buff_size, 813 struct i40e_asq_cmd_details *cmd_details) 814 { 815 enum i40e_status_code status = I40E_SUCCESS; 816 struct i40e_dma_mem *dma_buff = NULL; 817 struct i40e_asq_cmd_details *details; 818 struct i40e_aq_desc *desc_on_ring; 819 bool cmd_completed = false; 820 u16 retval = 0; 821 u32 val = 0; 822 823 i40e_acquire_spinlock(&hw->aq.asq_spinlock); 824 825 hw->aq.asq_last_status = I40E_AQ_RC_OK; 826 827 if (hw->aq.asq.count == 0) { 828 i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, 829 "AQTX: Admin queue not initialized.\n"); 830 status = I40E_ERR_QUEUE_EMPTY; 831 goto asq_send_command_error; 832 } 833 834 val = rd32(hw, hw->aq.asq.head); 835 if (val >= hw->aq.num_asq_entries) { 836 i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, 837 "AQTX: head overrun at %d\n", val); 838 status = I40E_ERR_QUEUE_EMPTY; 839 goto asq_send_command_error; 840 } 841 842 details = I40E_ADMINQ_DETAILS(hw->aq.asq, hw->aq.asq.next_to_use); 843 if (cmd_details) { 844 i40e_memcpy(details, 845 cmd_details, 846 sizeof(struct i40e_asq_cmd_details), 847 I40E_NONDMA_TO_NONDMA); 848 849 /* If the cmd_details are defined copy the cookie. The 850 * CPU_TO_LE32 is not needed here because the data is ignored 851 * by the FW, only used by the driver 852 */ 853 if (details->cookie) { 854 desc->cookie_high = 855 CPU_TO_LE32(I40E_HI_DWORD(details->cookie)); 856 desc->cookie_low = 857 CPU_TO_LE32(I40E_LO_DWORD(details->cookie)); 858 } 859 } else { 860 i40e_memset(details, 0, 861 sizeof(struct i40e_asq_cmd_details), 862 I40E_NONDMA_MEM); 863 } 864 865 /* clear requested flags and then set additional flags if defined */ 866 desc->flags &= ~CPU_TO_LE16(details->flags_dis); 867 desc->flags |= CPU_TO_LE16(details->flags_ena); 868 869 if (buff_size > hw->aq.asq_buf_size) { 870 i40e_debug(hw, 871 I40E_DEBUG_AQ_MESSAGE, 872 "AQTX: Invalid buffer size: %d.\n", 873 buff_size); 874 status = I40E_ERR_INVALID_SIZE; 875 goto asq_send_command_error; 876 } 877 878 if (details->postpone && !details->async) { 879 i40e_debug(hw, 880 I40E_DEBUG_AQ_MESSAGE, 881 "AQTX: Async flag not set along with postpone flag"); 882 status = I40E_ERR_PARAM; 883 goto asq_send_command_error; 884 } 885 886 /* call clean and check queue available function to reclaim the 887 * descriptors that were processed by FW, the function returns the 888 * number of desc available 889 */ 890 /* the clean function called here could be called in a separate thread 891 * in case of asynchronous completions 892 */ 893 if (i40e_clean_asq(hw) == 0) { 894 i40e_debug(hw, 895 I40E_DEBUG_AQ_MESSAGE, 896 "AQTX: Error queue is full.\n"); 897 status = I40E_ERR_ADMIN_QUEUE_FULL; 898 goto asq_send_command_error; 899 } 900 901 /* initialize the temp desc pointer with the right desc */ 902 desc_on_ring = I40E_ADMINQ_DESC(hw->aq.asq, hw->aq.asq.next_to_use); 903 904 /* if the desc is available copy the temp desc to the right place */ 905 i40e_memcpy(desc_on_ring, desc, sizeof(struct i40e_aq_desc), 906 I40E_NONDMA_TO_DMA); 907 908 /* if buff is not NULL assume indirect command */ 909 if (buff != NULL) { 910 dma_buff = &(hw->aq.asq.r.asq_bi[hw->aq.asq.next_to_use]); 911 /* copy the user buff into the respective DMA buff */ 912 i40e_memcpy(dma_buff->va, buff, buff_size, 913 I40E_NONDMA_TO_DMA); 914 desc_on_ring->datalen = CPU_TO_LE16(buff_size); 915 916 /* Update the address values in the desc with the pa value 917 * for respective buffer 918 */ 919 desc_on_ring->params.external.addr_high = 920 CPU_TO_LE32(I40E_HI_DWORD(dma_buff->pa)); 921 desc_on_ring->params.external.addr_low = 922 CPU_TO_LE32(I40E_LO_DWORD(dma_buff->pa)); 923 } 924 925 /* bump the tail */ 926 i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: desc and buffer:\n"); 927 i40e_debug_aq(hw, I40E_DEBUG_AQ_COMMAND, (void *)desc_on_ring, 928 buff, buff_size); 929 (hw->aq.asq.next_to_use)++; 930 if (hw->aq.asq.next_to_use == hw->aq.asq.count) 931 hw->aq.asq.next_to_use = 0; 932 if (!details->postpone) 933 wr32(hw, hw->aq.asq.tail, hw->aq.asq.next_to_use); 934 935 /* if cmd_details are not defined or async flag is not set, 936 * we need to wait for desc write back 937 */ 938 if (!details->async && !details->postpone) { 939 u32 total_delay = 0; 940 941 do { 942 /* AQ designers suggest use of head for better 943 * timing reliability than DD bit 944 */ 945 if (i40e_asq_done(hw)) 946 break; 947 i40e_usec_delay(50); 948 total_delay += 50; 949 } while (total_delay < hw->aq.asq_cmd_timeout); 950 } 951 952 /* if ready, copy the desc back to temp */ 953 if (i40e_asq_done(hw)) { 954 i40e_memcpy(desc, desc_on_ring, sizeof(struct i40e_aq_desc), 955 I40E_DMA_TO_NONDMA); 956 if (buff != NULL) 957 i40e_memcpy(buff, dma_buff->va, buff_size, 958 I40E_DMA_TO_NONDMA); 959 retval = LE16_TO_CPU(desc->retval); 960 if (retval != 0) { 961 i40e_debug(hw, 962 I40E_DEBUG_AQ_MESSAGE, 963 "AQTX: Command completed with error 0x%X.\n", 964 retval); 965 966 /* strip off FW internal code */ 967 retval &= 0xff; 968 } 969 cmd_completed = true; 970 if ((enum i40e_admin_queue_err)retval == I40E_AQ_RC_OK) 971 status = I40E_SUCCESS; 972 else if ((enum i40e_admin_queue_err)retval == I40E_AQ_RC_EBUSY) 973 status = I40E_ERR_NOT_READY; 974 else 975 status = I40E_ERR_ADMIN_QUEUE_ERROR; 976 hw->aq.asq_last_status = (enum i40e_admin_queue_err)retval; 977 } 978 979 i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, 980 "AQTX: desc and buffer writeback:\n"); 981 i40e_debug_aq(hw, I40E_DEBUG_AQ_COMMAND, (void *)desc, buff, buff_size); 982 983 /* save writeback aq if requested */ 984 if (details->wb_desc) 985 i40e_memcpy(details->wb_desc, desc_on_ring, 986 sizeof(struct i40e_aq_desc), I40E_DMA_TO_NONDMA); 987 988 /* update the error if time out occurred */ 989 if ((!cmd_completed) && 990 (!details->async && !details->postpone)) { 991 #ifdef PF_DRIVER 992 if (rd32(hw, hw->aq.asq.len) & I40E_GL_ATQLEN_ATQCRIT_MASK) { 993 #else 994 if (rd32(hw, hw->aq.asq.len) & I40E_VF_ATQLEN1_ATQCRIT_MASK) { 995 #endif 996 i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, 997 "AQTX: AQ Critical error.\n"); 998 status = I40E_ERR_ADMIN_QUEUE_CRITICAL_ERROR; 999 } else { 1000 i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, 1001 "AQTX: Writeback timeout.\n"); 1002 status = I40E_ERR_ADMIN_QUEUE_TIMEOUT; 1003 } 1004 } 1005 1006 asq_send_command_error: 1007 i40e_release_spinlock(&hw->aq.asq_spinlock); 1008 return status; 1009 } 1010 1011 /** 1012 * i40e_fill_default_direct_cmd_desc - AQ descriptor helper function 1013 * @desc: pointer to the temp descriptor (non DMA mem) 1014 * @opcode: the opcode can be used to decide which flags to turn off or on 1015 * 1016 * Fill the desc with default values 1017 **/ 1018 void i40e_fill_default_direct_cmd_desc(struct i40e_aq_desc *desc, 1019 u16 opcode) 1020 { 1021 /* zero out the desc */ 1022 i40e_memset((void *)desc, 0, sizeof(struct i40e_aq_desc), 1023 I40E_NONDMA_MEM); 1024 desc->opcode = CPU_TO_LE16(opcode); 1025 desc->flags = CPU_TO_LE16(I40E_AQ_FLAG_SI); 1026 } 1027 1028 /** 1029 * i40e_clean_arq_element 1030 * @hw: pointer to the hw struct 1031 * @e: event info from the receive descriptor, includes any buffers 1032 * @pending: number of events that could be left to process 1033 * 1034 * This function cleans one Admin Receive Queue element and returns 1035 * the contents through e. It can also return how many events are 1036 * left to process through 'pending' 1037 **/ 1038 enum i40e_status_code i40e_clean_arq_element(struct i40e_hw *hw, 1039 struct i40e_arq_event_info *e, 1040 u16 *pending) 1041 { 1042 enum i40e_status_code ret_code = I40E_SUCCESS; 1043 u16 ntc = hw->aq.arq.next_to_clean; 1044 struct i40e_aq_desc *desc; 1045 struct i40e_dma_mem *bi; 1046 u16 desc_idx; 1047 u16 datalen; 1048 u16 flags; 1049 u16 ntu; 1050 1051 /* pre-clean the event info */ 1052 i40e_memset(&e->desc, 0, sizeof(e->desc), I40E_NONDMA_MEM); 1053 1054 /* take the lock before we start messing with the ring */ 1055 i40e_acquire_spinlock(&hw->aq.arq_spinlock); 1056 1057 if (hw->aq.arq.count == 0) { 1058 i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, 1059 "AQRX: Admin queue not initialized.\n"); 1060 ret_code = I40E_ERR_QUEUE_EMPTY; 1061 goto clean_arq_element_err; 1062 } 1063 1064 /* set next_to_use to head */ 1065 #ifdef INTEGRATED_VF 1066 if (!i40e_is_vf(hw)) 1067 ntu = rd32(hw, hw->aq.arq.head) & I40E_PF_ARQH_ARQH_MASK; 1068 else 1069 ntu = rd32(hw, hw->aq.arq.head) & I40E_VF_ARQH1_ARQH_MASK; 1070 #else 1071 #ifdef PF_DRIVER 1072 ntu = rd32(hw, hw->aq.arq.head) & I40E_PF_ARQH_ARQH_MASK; 1073 #endif /* PF_DRIVER */ 1074 #ifdef VF_DRIVER 1075 ntu = rd32(hw, hw->aq.arq.head) & I40E_VF_ARQH1_ARQH_MASK; 1076 #endif /* VF_DRIVER */ 1077 #endif /* INTEGRATED_VF */ 1078 if (ntu == ntc) { 1079 /* nothing to do - shouldn't need to update ring's values */ 1080 ret_code = I40E_ERR_ADMIN_QUEUE_NO_WORK; 1081 goto clean_arq_element_out; 1082 } 1083 1084 /* now clean the next descriptor */ 1085 desc = I40E_ADMINQ_DESC(hw->aq.arq, ntc); 1086 desc_idx = ntc; 1087 1088 hw->aq.arq_last_status = 1089 (enum i40e_admin_queue_err)LE16_TO_CPU(desc->retval); 1090 flags = LE16_TO_CPU(desc->flags); 1091 if (flags & I40E_AQ_FLAG_ERR) { 1092 ret_code = I40E_ERR_ADMIN_QUEUE_ERROR; 1093 i40e_debug(hw, 1094 I40E_DEBUG_AQ_MESSAGE, 1095 "AQRX: Event received with error 0x%X.\n", 1096 hw->aq.arq_last_status); 1097 } 1098 1099 i40e_memcpy(&e->desc, desc, sizeof(struct i40e_aq_desc), 1100 I40E_DMA_TO_NONDMA); 1101 datalen = LE16_TO_CPU(desc->datalen); 1102 e->msg_len = min(datalen, e->buf_len); 1103 if (e->msg_buf != NULL && (e->msg_len != 0)) 1104 i40e_memcpy(e->msg_buf, 1105 hw->aq.arq.r.arq_bi[desc_idx].va, 1106 e->msg_len, I40E_DMA_TO_NONDMA); 1107 1108 i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQRX: desc and buffer:\n"); 1109 i40e_debug_aq(hw, I40E_DEBUG_AQ_COMMAND, (void *)desc, e->msg_buf, 1110 hw->aq.arq_buf_size); 1111 1112 /* Restore the original datalen and buffer address in the desc, 1113 * FW updates datalen to indicate the event message 1114 * size 1115 */ 1116 bi = &hw->aq.arq.r.arq_bi[ntc]; 1117 i40e_memset((void *)desc, 0, sizeof(struct i40e_aq_desc), I40E_DMA_MEM); 1118 1119 desc->flags = CPU_TO_LE16(I40E_AQ_FLAG_BUF); 1120 if (hw->aq.arq_buf_size > I40E_AQ_LARGE_BUF) 1121 desc->flags |= CPU_TO_LE16(I40E_AQ_FLAG_LB); 1122 desc->datalen = CPU_TO_LE16((u16)bi->size); 1123 desc->params.external.addr_high = CPU_TO_LE32(I40E_HI_DWORD(bi->pa)); 1124 desc->params.external.addr_low = CPU_TO_LE32(I40E_LO_DWORD(bi->pa)); 1125 1126 /* set tail = the last cleaned desc index. */ 1127 wr32(hw, hw->aq.arq.tail, ntc); 1128 /* ntc is updated to tail + 1 */ 1129 ntc++; 1130 if (ntc == hw->aq.num_arq_entries) 1131 ntc = 0; 1132 hw->aq.arq.next_to_clean = ntc; 1133 hw->aq.arq.next_to_use = ntu; 1134 1135 #ifdef PF_DRIVER 1136 i40e_nvmupd_check_wait_event(hw, LE16_TO_CPU(e->desc.opcode), &e->desc); 1137 #endif /* PF_DRIVER */ 1138 clean_arq_element_out: 1139 /* Set pending if needed, unlock and return */ 1140 if (pending != NULL) 1141 *pending = (ntc > ntu ? hw->aq.arq.count : 0) + (ntu - ntc); 1142 clean_arq_element_err: 1143 i40e_release_spinlock(&hw->aq.arq_spinlock); 1144 1145 return ret_code; 1146 } 1147 1148