1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates. 3 * All rights reserved. 4 */ 5 6 #include "ena_com.h" 7 8 /*****************************************************************************/ 9 /*****************************************************************************/ 10 11 /* Timeout in micro-sec */ 12 #define ADMIN_CMD_TIMEOUT_US (3000000) 13 14 #define ENA_ASYNC_QUEUE_DEPTH 16 15 #define ENA_ADMIN_QUEUE_DEPTH 32 16 17 #define ENA_CTRL_MAJOR 0 18 #define ENA_CTRL_MINOR 0 19 #define ENA_CTRL_SUB_MINOR 1 20 21 #define MIN_ENA_CTRL_VER \ 22 (((ENA_CTRL_MAJOR) << \ 23 (ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT)) | \ 24 ((ENA_CTRL_MINOR) << \ 25 (ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT)) | \ 26 (ENA_CTRL_SUB_MINOR)) 27 28 #define ENA_DMA_ADDR_TO_UINT32_LOW(x) ((u32)((u64)(x))) 29 #define ENA_DMA_ADDR_TO_UINT32_HIGH(x) ((u32)(((u64)(x)) >> 32)) 30 31 #define ENA_MMIO_READ_TIMEOUT 0xFFFFFFFF 32 33 #define ENA_COM_BOUNCE_BUFFER_CNTRL_CNT 4 34 35 #define ENA_REGS_ADMIN_INTR_MASK 1 36 37 #define ENA_MIN_ADMIN_POLL_US 100 38 39 #define ENA_MAX_ADMIN_POLL_US 5000 40 41 /*****************************************************************************/ 42 /*****************************************************************************/ 43 /*****************************************************************************/ 44 45 enum ena_cmd_status { 46 ENA_CMD_SUBMITTED, 47 ENA_CMD_COMPLETED, 48 /* Abort - canceled by the driver */ 49 ENA_CMD_ABORTED, 50 }; 51 52 struct ena_comp_ctx { 53 ena_wait_event_t wait_event; 54 struct ena_admin_acq_entry *user_cqe; 55 u32 comp_size; 56 enum ena_cmd_status status; 57 /* status from the device */ 58 u8 comp_status; 59 u8 cmd_opcode; 60 bool occupied; 61 }; 62 63 struct ena_com_stats_ctx { 64 struct ena_admin_aq_get_stats_cmd get_cmd; 65 struct ena_admin_acq_get_stats_resp get_resp; 66 }; 67 68 static int ena_com_mem_addr_set(struct ena_com_dev *ena_dev, 69 struct ena_common_mem_addr *ena_addr, 70 dma_addr_t addr) 71 { 72 if ((addr & GENMASK_ULL(ena_dev->dma_addr_bits - 1, 0)) != addr) { 73 ena_trc_err(ena_dev, "DMA address has more bits than the device supports\n"); 74 return ENA_COM_INVAL; 75 } 76 77 ena_addr->mem_addr_low = lower_32_bits(addr); 78 ena_addr->mem_addr_high = (u16)upper_32_bits(addr); 79 80 return 0; 81 } 82 83 static int ena_com_admin_init_sq(struct ena_com_admin_queue *admin_queue) 84 { 85 struct ena_com_dev *ena_dev = admin_queue->ena_dev; 86 struct ena_com_admin_sq *sq = &admin_queue->sq; 87 u16 size = ADMIN_SQ_SIZE(admin_queue->q_depth); 88 89 ENA_MEM_ALLOC_COHERENT(admin_queue->q_dmadev, size, sq->entries, sq->dma_addr, 90 sq->mem_handle); 91 92 if (!sq->entries) { 93 ena_trc_err(ena_dev, "Memory allocation failed\n"); 94 return ENA_COM_NO_MEM; 95 } 96 97 sq->head = 0; 98 sq->tail = 0; 99 sq->phase = 1; 100 101 sq->db_addr = NULL; 102 103 return 0; 104 } 105 106 static int ena_com_admin_init_cq(struct ena_com_admin_queue *admin_queue) 107 { 108 struct ena_com_dev *ena_dev = admin_queue->ena_dev; 109 struct ena_com_admin_cq *cq = &admin_queue->cq; 110 u16 size = ADMIN_CQ_SIZE(admin_queue->q_depth); 111 112 ENA_MEM_ALLOC_COHERENT(admin_queue->q_dmadev, size, cq->entries, cq->dma_addr, 113 cq->mem_handle); 114 115 if (!cq->entries) { 116 ena_trc_err(ena_dev, "Memory allocation failed\n"); 117 return ENA_COM_NO_MEM; 118 } 119 120 cq->head = 0; 121 cq->phase = 1; 122 123 return 0; 124 } 125 126 static int ena_com_admin_init_aenq(struct ena_com_dev *ena_dev, 127 struct ena_aenq_handlers *aenq_handlers) 128 { 129 struct ena_com_aenq *aenq = &ena_dev->aenq; 130 u32 addr_low, addr_high, aenq_caps; 131 u16 size; 132 133 ena_dev->aenq.q_depth = ENA_ASYNC_QUEUE_DEPTH; 134 size = ADMIN_AENQ_SIZE(ENA_ASYNC_QUEUE_DEPTH); 135 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, size, 136 aenq->entries, 137 aenq->dma_addr, 138 aenq->mem_handle); 139 140 if (!aenq->entries) { 141 ena_trc_err(ena_dev, "Memory allocation failed\n"); 142 return ENA_COM_NO_MEM; 143 } 144 145 aenq->head = aenq->q_depth; 146 aenq->phase = 1; 147 148 addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(aenq->dma_addr); 149 addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(aenq->dma_addr); 150 151 ENA_REG_WRITE32(ena_dev->bus, addr_low, ena_dev->reg_bar + ENA_REGS_AENQ_BASE_LO_OFF); 152 ENA_REG_WRITE32(ena_dev->bus, addr_high, ena_dev->reg_bar + ENA_REGS_AENQ_BASE_HI_OFF); 153 154 aenq_caps = 0; 155 aenq_caps |= ena_dev->aenq.q_depth & ENA_REGS_AENQ_CAPS_AENQ_DEPTH_MASK; 156 aenq_caps |= (sizeof(struct ena_admin_aenq_entry) << 157 ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_SHIFT) & 158 ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_MASK; 159 ENA_REG_WRITE32(ena_dev->bus, aenq_caps, ena_dev->reg_bar + ENA_REGS_AENQ_CAPS_OFF); 160 161 if (unlikely(!aenq_handlers)) { 162 ena_trc_err(ena_dev, "AENQ handlers pointer is NULL\n"); 163 return ENA_COM_INVAL; 164 } 165 166 aenq->aenq_handlers = aenq_handlers; 167 168 return 0; 169 } 170 171 static void comp_ctxt_release(struct ena_com_admin_queue *queue, 172 struct ena_comp_ctx *comp_ctx) 173 { 174 comp_ctx->occupied = false; 175 ATOMIC32_DEC(&queue->outstanding_cmds); 176 } 177 178 static struct ena_comp_ctx *get_comp_ctxt(struct ena_com_admin_queue *admin_queue, 179 u16 command_id, bool capture) 180 { 181 if (unlikely(command_id >= admin_queue->q_depth)) { 182 ena_trc_err(admin_queue->ena_dev, 183 "Command id is larger than the queue size. cmd_id: %u queue size %d\n", 184 command_id, admin_queue->q_depth); 185 return NULL; 186 } 187 188 if (unlikely(!admin_queue->comp_ctx)) { 189 ena_trc_err(admin_queue->ena_dev, 190 "Completion context is NULL\n"); 191 return NULL; 192 } 193 194 if (unlikely(admin_queue->comp_ctx[command_id].occupied && capture)) { 195 ena_trc_err(admin_queue->ena_dev, 196 "Completion context is occupied\n"); 197 return NULL; 198 } 199 200 if (capture) { 201 ATOMIC32_INC(&admin_queue->outstanding_cmds); 202 admin_queue->comp_ctx[command_id].occupied = true; 203 } 204 205 return &admin_queue->comp_ctx[command_id]; 206 } 207 208 static struct ena_comp_ctx *__ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue, 209 struct ena_admin_aq_entry *cmd, 210 size_t cmd_size_in_bytes, 211 struct ena_admin_acq_entry *comp, 212 size_t comp_size_in_bytes) 213 { 214 struct ena_comp_ctx *comp_ctx; 215 u16 tail_masked, cmd_id; 216 u16 queue_size_mask; 217 u16 cnt; 218 219 queue_size_mask = admin_queue->q_depth - 1; 220 221 tail_masked = admin_queue->sq.tail & queue_size_mask; 222 223 /* In case of queue FULL */ 224 cnt = (u16)ATOMIC32_READ(&admin_queue->outstanding_cmds); 225 if (cnt >= admin_queue->q_depth) { 226 ena_trc_dbg(admin_queue->ena_dev, "Admin queue is full.\n"); 227 admin_queue->stats.out_of_space++; 228 return ERR_PTR(ENA_COM_NO_SPACE); 229 } 230 231 cmd_id = admin_queue->curr_cmd_id; 232 233 cmd->aq_common_descriptor.flags |= admin_queue->sq.phase & 234 ENA_ADMIN_AQ_COMMON_DESC_PHASE_MASK; 235 236 cmd->aq_common_descriptor.command_id |= cmd_id & 237 ENA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK; 238 239 comp_ctx = get_comp_ctxt(admin_queue, cmd_id, true); 240 if (unlikely(!comp_ctx)) 241 return ERR_PTR(ENA_COM_INVAL); 242 243 comp_ctx->status = ENA_CMD_SUBMITTED; 244 comp_ctx->comp_size = (u32)comp_size_in_bytes; 245 comp_ctx->user_cqe = comp; 246 comp_ctx->cmd_opcode = cmd->aq_common_descriptor.opcode; 247 248 ENA_WAIT_EVENT_CLEAR(comp_ctx->wait_event); 249 250 memcpy(&admin_queue->sq.entries[tail_masked], cmd, cmd_size_in_bytes); 251 252 admin_queue->curr_cmd_id = (admin_queue->curr_cmd_id + 1) & 253 queue_size_mask; 254 255 admin_queue->sq.tail++; 256 admin_queue->stats.submitted_cmd++; 257 258 if (unlikely((admin_queue->sq.tail & queue_size_mask) == 0)) 259 admin_queue->sq.phase = !admin_queue->sq.phase; 260 261 ENA_DB_SYNC(&admin_queue->sq.mem_handle); 262 ENA_REG_WRITE32(admin_queue->bus, admin_queue->sq.tail, 263 admin_queue->sq.db_addr); 264 265 return comp_ctx; 266 } 267 268 static int ena_com_init_comp_ctxt(struct ena_com_admin_queue *admin_queue) 269 { 270 struct ena_com_dev *ena_dev = admin_queue->ena_dev; 271 size_t size = admin_queue->q_depth * sizeof(struct ena_comp_ctx); 272 struct ena_comp_ctx *comp_ctx; 273 u16 i; 274 275 admin_queue->comp_ctx = ENA_MEM_ALLOC(admin_queue->q_dmadev, size); 276 if (unlikely(!admin_queue->comp_ctx)) { 277 ena_trc_err(ena_dev, "Memory allocation failed\n"); 278 return ENA_COM_NO_MEM; 279 } 280 281 for (i = 0; i < admin_queue->q_depth; i++) { 282 comp_ctx = get_comp_ctxt(admin_queue, i, false); 283 if (comp_ctx) 284 ENA_WAIT_EVENT_INIT(comp_ctx->wait_event); 285 } 286 287 return 0; 288 } 289 290 static struct ena_comp_ctx *ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue, 291 struct ena_admin_aq_entry *cmd, 292 size_t cmd_size_in_bytes, 293 struct ena_admin_acq_entry *comp, 294 size_t comp_size_in_bytes) 295 { 296 unsigned long flags = 0; 297 struct ena_comp_ctx *comp_ctx; 298 299 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); 300 if (unlikely(!admin_queue->running_state)) { 301 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); 302 return ERR_PTR(ENA_COM_NO_DEVICE); 303 } 304 comp_ctx = __ena_com_submit_admin_cmd(admin_queue, cmd, 305 cmd_size_in_bytes, 306 comp, 307 comp_size_in_bytes); 308 if (IS_ERR(comp_ctx)) 309 admin_queue->running_state = false; 310 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); 311 312 return comp_ctx; 313 } 314 315 static int ena_com_init_io_sq(struct ena_com_dev *ena_dev, 316 struct ena_com_create_io_ctx *ctx, 317 struct ena_com_io_sq *io_sq) 318 { 319 size_t size; 320 int dev_node = 0; 321 322 memset(&io_sq->desc_addr, 0x0, sizeof(io_sq->desc_addr)); 323 324 io_sq->dma_addr_bits = (u8)ena_dev->dma_addr_bits; 325 io_sq->desc_entry_size = 326 (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) ? 327 sizeof(struct ena_eth_io_tx_desc) : 328 sizeof(struct ena_eth_io_rx_desc); 329 330 size = io_sq->desc_entry_size * io_sq->q_depth; 331 io_sq->bus = ena_dev->bus; 332 333 if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) { 334 ENA_MEM_ALLOC_COHERENT_NODE(ena_dev->dmadev, 335 size, 336 io_sq->desc_addr.virt_addr, 337 io_sq->desc_addr.phys_addr, 338 io_sq->desc_addr.mem_handle, 339 ctx->numa_node, 340 dev_node); 341 if (!io_sq->desc_addr.virt_addr) { 342 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, 343 size, 344 io_sq->desc_addr.virt_addr, 345 io_sq->desc_addr.phys_addr, 346 io_sq->desc_addr.mem_handle); 347 } 348 349 if (!io_sq->desc_addr.virt_addr) { 350 ena_trc_err(ena_dev, "Memory allocation failed\n"); 351 return ENA_COM_NO_MEM; 352 } 353 } 354 355 if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { 356 /* Allocate bounce buffers */ 357 io_sq->bounce_buf_ctrl.buffer_size = 358 ena_dev->llq_info.desc_list_entry_size; 359 io_sq->bounce_buf_ctrl.buffers_num = 360 ENA_COM_BOUNCE_BUFFER_CNTRL_CNT; 361 io_sq->bounce_buf_ctrl.next_to_use = 0; 362 363 size = io_sq->bounce_buf_ctrl.buffer_size * 364 io_sq->bounce_buf_ctrl.buffers_num; 365 366 ENA_MEM_ALLOC_NODE(ena_dev->dmadev, 367 size, 368 io_sq->bounce_buf_ctrl.base_buffer, 369 ctx->numa_node, 370 dev_node); 371 if (!io_sq->bounce_buf_ctrl.base_buffer) 372 io_sq->bounce_buf_ctrl.base_buffer = ENA_MEM_ALLOC(ena_dev->dmadev, size); 373 374 if (!io_sq->bounce_buf_ctrl.base_buffer) { 375 ena_trc_err(ena_dev, "Bounce buffer memory allocation failed\n"); 376 return ENA_COM_NO_MEM; 377 } 378 379 memcpy(&io_sq->llq_info, &ena_dev->llq_info, 380 sizeof(io_sq->llq_info)); 381 382 /* Initiate the first bounce buffer */ 383 io_sq->llq_buf_ctrl.curr_bounce_buf = 384 ena_com_get_next_bounce_buffer(&io_sq->bounce_buf_ctrl); 385 memset(io_sq->llq_buf_ctrl.curr_bounce_buf, 386 0x0, io_sq->llq_info.desc_list_entry_size); 387 io_sq->llq_buf_ctrl.descs_left_in_line = 388 io_sq->llq_info.descs_num_before_header; 389 io_sq->disable_meta_caching = 390 io_sq->llq_info.disable_meta_caching; 391 392 if (io_sq->llq_info.max_entries_in_tx_burst > 0) 393 io_sq->entries_in_tx_burst_left = 394 io_sq->llq_info.max_entries_in_tx_burst; 395 } 396 397 io_sq->tail = 0; 398 io_sq->next_to_comp = 0; 399 io_sq->phase = 1; 400 401 return 0; 402 } 403 404 static int ena_com_init_io_cq(struct ena_com_dev *ena_dev, 405 struct ena_com_create_io_ctx *ctx, 406 struct ena_com_io_cq *io_cq) 407 { 408 size_t size; 409 int prev_node = 0; 410 411 memset(&io_cq->cdesc_addr, 0x0, sizeof(io_cq->cdesc_addr)); 412 413 /* Use the basic completion descriptor for Rx */ 414 io_cq->cdesc_entry_size_in_bytes = 415 (io_cq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) ? 416 sizeof(struct ena_eth_io_tx_cdesc) : 417 sizeof(struct ena_eth_io_rx_cdesc_base); 418 419 size = io_cq->cdesc_entry_size_in_bytes * io_cq->q_depth; 420 io_cq->bus = ena_dev->bus; 421 422 ENA_MEM_ALLOC_COHERENT_NODE_ALIGNED(ena_dev->dmadev, 423 size, 424 io_cq->cdesc_addr.virt_addr, 425 io_cq->cdesc_addr.phys_addr, 426 io_cq->cdesc_addr.mem_handle, 427 ctx->numa_node, 428 prev_node, 429 ENA_CDESC_RING_SIZE_ALIGNMENT); 430 if (!io_cq->cdesc_addr.virt_addr) { 431 ENA_MEM_ALLOC_COHERENT_ALIGNED(ena_dev->dmadev, 432 size, 433 io_cq->cdesc_addr.virt_addr, 434 io_cq->cdesc_addr.phys_addr, 435 io_cq->cdesc_addr.mem_handle, 436 ENA_CDESC_RING_SIZE_ALIGNMENT); 437 } 438 439 if (!io_cq->cdesc_addr.virt_addr) { 440 ena_trc_err(ena_dev, "Memory allocation failed\n"); 441 return ENA_COM_NO_MEM; 442 } 443 444 io_cq->phase = 1; 445 io_cq->head = 0; 446 447 return 0; 448 } 449 450 static void ena_com_handle_single_admin_completion(struct ena_com_admin_queue *admin_queue, 451 struct ena_admin_acq_entry *cqe) 452 { 453 struct ena_comp_ctx *comp_ctx; 454 u16 cmd_id; 455 456 cmd_id = cqe->acq_common_descriptor.command & 457 ENA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK; 458 459 comp_ctx = get_comp_ctxt(admin_queue, cmd_id, false); 460 if (unlikely(!comp_ctx)) { 461 ena_trc_err(admin_queue->ena_dev, 462 "comp_ctx is NULL. Changing the admin queue running state\n"); 463 admin_queue->running_state = false; 464 return; 465 } 466 467 comp_ctx->status = ENA_CMD_COMPLETED; 468 comp_ctx->comp_status = cqe->acq_common_descriptor.status; 469 470 if (comp_ctx->user_cqe) 471 memcpy(comp_ctx->user_cqe, (void *)cqe, comp_ctx->comp_size); 472 473 if (!admin_queue->polling) 474 ENA_WAIT_EVENT_SIGNAL(comp_ctx->wait_event); 475 } 476 477 static void ena_com_handle_admin_completion(struct ena_com_admin_queue *admin_queue) 478 { 479 struct ena_admin_acq_entry *cqe = NULL; 480 u16 comp_num = 0; 481 u16 head_masked; 482 u8 phase; 483 484 head_masked = admin_queue->cq.head & (admin_queue->q_depth - 1); 485 phase = admin_queue->cq.phase; 486 487 cqe = &admin_queue->cq.entries[head_masked]; 488 489 /* Go over all the completions */ 490 while ((READ_ONCE8(cqe->acq_common_descriptor.flags) & 491 ENA_ADMIN_ACQ_COMMON_DESC_PHASE_MASK) == phase) { 492 /* Do not read the rest of the completion entry before the 493 * phase bit was validated 494 */ 495 dma_rmb(); 496 ena_com_handle_single_admin_completion(admin_queue, cqe); 497 498 head_masked++; 499 comp_num++; 500 if (unlikely(head_masked == admin_queue->q_depth)) { 501 head_masked = 0; 502 phase = !phase; 503 } 504 505 cqe = &admin_queue->cq.entries[head_masked]; 506 } 507 508 admin_queue->cq.head += comp_num; 509 admin_queue->cq.phase = phase; 510 admin_queue->sq.head += comp_num; 511 admin_queue->stats.completed_cmd += comp_num; 512 } 513 514 static int ena_com_comp_status_to_errno(struct ena_com_admin_queue *admin_queue, 515 u8 comp_status) 516 { 517 if (unlikely(comp_status != 0)) 518 ena_trc_err(admin_queue->ena_dev, 519 "Admin command failed[%u]\n", comp_status); 520 521 switch (comp_status) { 522 case ENA_ADMIN_SUCCESS: 523 return ENA_COM_OK; 524 case ENA_ADMIN_RESOURCE_ALLOCATION_FAILURE: 525 return ENA_COM_NO_MEM; 526 case ENA_ADMIN_UNSUPPORTED_OPCODE: 527 return ENA_COM_UNSUPPORTED; 528 case ENA_ADMIN_BAD_OPCODE: 529 case ENA_ADMIN_MALFORMED_REQUEST: 530 case ENA_ADMIN_ILLEGAL_PARAMETER: 531 case ENA_ADMIN_UNKNOWN_ERROR: 532 return ENA_COM_INVAL; 533 case ENA_ADMIN_RESOURCE_BUSY: 534 return ENA_COM_TRY_AGAIN; 535 } 536 537 return ENA_COM_INVAL; 538 } 539 540 static void ena_delay_exponential_backoff_us(u32 exp, u32 delay_us) 541 { 542 delay_us = ENA_MAX32(ENA_MIN_ADMIN_POLL_US, delay_us); 543 delay_us = ENA_MIN32(delay_us * (1U << exp), ENA_MAX_ADMIN_POLL_US); 544 ENA_USLEEP(delay_us); 545 } 546 547 static int ena_com_wait_and_process_admin_cq_polling(struct ena_comp_ctx *comp_ctx, 548 struct ena_com_admin_queue *admin_queue) 549 { 550 unsigned long flags = 0; 551 ena_time_t timeout; 552 int ret; 553 u32 exp = 0; 554 555 timeout = ENA_GET_SYSTEM_TIMEOUT(admin_queue->completion_timeout); 556 557 while (1) { 558 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); 559 ena_com_handle_admin_completion(admin_queue); 560 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); 561 562 if (comp_ctx->status != ENA_CMD_SUBMITTED) 563 break; 564 565 if (ENA_TIME_EXPIRE(timeout)) { 566 ena_trc_err(admin_queue->ena_dev, 567 "Wait for completion (polling) timeout\n"); 568 /* ENA didn't have any completion */ 569 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); 570 admin_queue->stats.no_completion++; 571 admin_queue->running_state = false; 572 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); 573 574 ret = ENA_COM_TIMER_EXPIRED; 575 goto err; 576 } 577 578 ena_delay_exponential_backoff_us(exp++, 579 admin_queue->ena_dev->ena_min_poll_delay_us); 580 } 581 582 if (unlikely(comp_ctx->status == ENA_CMD_ABORTED)) { 583 ena_trc_err(admin_queue->ena_dev, "Command was aborted\n"); 584 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); 585 admin_queue->stats.aborted_cmd++; 586 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); 587 ret = ENA_COM_NO_DEVICE; 588 goto err; 589 } 590 591 ENA_WARN(comp_ctx->status != ENA_CMD_COMPLETED, 592 admin_queue->ena_dev, "Invalid comp status %d\n", 593 comp_ctx->status); 594 595 ret = ena_com_comp_status_to_errno(admin_queue, comp_ctx->comp_status); 596 err: 597 comp_ctxt_release(admin_queue, comp_ctx); 598 return ret; 599 } 600 601 /* 602 * Set the LLQ configurations of the firmware 603 * 604 * The driver provides only the enabled feature values to the device, 605 * which in turn, checks if they are supported. 606 */ 607 static int ena_com_set_llq(struct ena_com_dev *ena_dev) 608 { 609 struct ena_com_admin_queue *admin_queue; 610 struct ena_admin_set_feat_cmd cmd; 611 struct ena_admin_set_feat_resp resp; 612 struct ena_com_llq_info *llq_info = &ena_dev->llq_info; 613 int ret; 614 615 memset(&cmd, 0x0, sizeof(cmd)); 616 admin_queue = &ena_dev->admin_queue; 617 618 cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE; 619 cmd.feat_common.feature_id = ENA_ADMIN_LLQ; 620 621 cmd.u.llq.header_location_ctrl_enabled = llq_info->header_location_ctrl; 622 cmd.u.llq.entry_size_ctrl_enabled = llq_info->desc_list_entry_size_ctrl; 623 cmd.u.llq.desc_num_before_header_enabled = llq_info->descs_num_before_header; 624 cmd.u.llq.descriptors_stride_ctrl_enabled = llq_info->desc_stride_ctrl; 625 626 cmd.u.llq.accel_mode.u.set.enabled_flags = 627 BIT(ENA_ADMIN_DISABLE_META_CACHING) | 628 BIT(ENA_ADMIN_LIMIT_TX_BURST); 629 630 ret = ena_com_execute_admin_command(admin_queue, 631 (struct ena_admin_aq_entry *)&cmd, 632 sizeof(cmd), 633 (struct ena_admin_acq_entry *)&resp, 634 sizeof(resp)); 635 636 if (unlikely(ret)) 637 ena_trc_err(ena_dev, "Failed to set LLQ configurations: %d\n", ret); 638 639 return ret; 640 } 641 642 static int ena_com_config_llq_info(struct ena_com_dev *ena_dev, 643 struct ena_admin_feature_llq_desc *llq_features, 644 struct ena_llq_configurations *llq_default_cfg) 645 { 646 struct ena_com_llq_info *llq_info = &ena_dev->llq_info; 647 struct ena_admin_accel_mode_get llq_accel_mode_get; 648 u16 supported_feat; 649 int rc; 650 651 memset(llq_info, 0, sizeof(*llq_info)); 652 653 supported_feat = llq_features->header_location_ctrl_supported; 654 655 if (likely(supported_feat & llq_default_cfg->llq_header_location)) { 656 llq_info->header_location_ctrl = 657 llq_default_cfg->llq_header_location; 658 } else { 659 ena_trc_err(ena_dev, "Invalid header location control, supported: 0x%x\n", 660 supported_feat); 661 return -EINVAL; 662 } 663 664 if (likely(llq_info->header_location_ctrl == ENA_ADMIN_INLINE_HEADER)) { 665 supported_feat = llq_features->descriptors_stride_ctrl_supported; 666 if (likely(supported_feat & llq_default_cfg->llq_stride_ctrl)) { 667 llq_info->desc_stride_ctrl = llq_default_cfg->llq_stride_ctrl; 668 } else { 669 if (supported_feat & ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY) { 670 llq_info->desc_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY; 671 } else if (supported_feat & ENA_ADMIN_SINGLE_DESC_PER_ENTRY) { 672 llq_info->desc_stride_ctrl = ENA_ADMIN_SINGLE_DESC_PER_ENTRY; 673 } else { 674 ena_trc_err(ena_dev, "Invalid desc_stride_ctrl, supported: 0x%x\n", 675 supported_feat); 676 return -EINVAL; 677 } 678 679 ena_trc_err(ena_dev, "Default llq stride ctrl is not supported, performing fallback, default: 0x%x, supported: 0x%x, used: 0x%x\n", 680 llq_default_cfg->llq_stride_ctrl, 681 supported_feat, 682 llq_info->desc_stride_ctrl); 683 } 684 } else { 685 llq_info->desc_stride_ctrl = 0; 686 } 687 688 supported_feat = llq_features->entry_size_ctrl_supported; 689 if (likely(supported_feat & llq_default_cfg->llq_ring_entry_size)) { 690 llq_info->desc_list_entry_size_ctrl = llq_default_cfg->llq_ring_entry_size; 691 llq_info->desc_list_entry_size = llq_default_cfg->llq_ring_entry_size_value; 692 } else { 693 if (supported_feat & ENA_ADMIN_LIST_ENTRY_SIZE_128B) { 694 llq_info->desc_list_entry_size_ctrl = ENA_ADMIN_LIST_ENTRY_SIZE_128B; 695 llq_info->desc_list_entry_size = 128; 696 } else if (supported_feat & ENA_ADMIN_LIST_ENTRY_SIZE_192B) { 697 llq_info->desc_list_entry_size_ctrl = ENA_ADMIN_LIST_ENTRY_SIZE_192B; 698 llq_info->desc_list_entry_size = 192; 699 } else if (supported_feat & ENA_ADMIN_LIST_ENTRY_SIZE_256B) { 700 llq_info->desc_list_entry_size_ctrl = ENA_ADMIN_LIST_ENTRY_SIZE_256B; 701 llq_info->desc_list_entry_size = 256; 702 } else { 703 ena_trc_err(ena_dev, "Invalid entry_size_ctrl, supported: 0x%x\n", 704 supported_feat); 705 return -EINVAL; 706 } 707 708 ena_trc_err(ena_dev, "Default llq ring entry size is not supported, performing fallback, default: 0x%x, supported: 0x%x, used: 0x%x\n", 709 llq_default_cfg->llq_ring_entry_size, 710 supported_feat, 711 llq_info->desc_list_entry_size); 712 } 713 if (unlikely(llq_info->desc_list_entry_size & 0x7)) { 714 /* The desc list entry size should be whole multiply of 8 715 * This requirement comes from __iowrite64_copy() 716 */ 717 ena_trc_err(ena_dev, "Illegal entry size %d\n", 718 llq_info->desc_list_entry_size); 719 return -EINVAL; 720 } 721 722 if (llq_info->desc_stride_ctrl == ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY) 723 llq_info->descs_per_entry = llq_info->desc_list_entry_size / 724 sizeof(struct ena_eth_io_tx_desc); 725 else 726 llq_info->descs_per_entry = 1; 727 728 supported_feat = llq_features->desc_num_before_header_supported; 729 if (likely(supported_feat & llq_default_cfg->llq_num_decs_before_header)) { 730 llq_info->descs_num_before_header = llq_default_cfg->llq_num_decs_before_header; 731 } else { 732 if (supported_feat & ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2) { 733 llq_info->descs_num_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2; 734 } else if (supported_feat & ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_1) { 735 llq_info->descs_num_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_1; 736 } else if (supported_feat & ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_4) { 737 llq_info->descs_num_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_4; 738 } else if (supported_feat & ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_8) { 739 llq_info->descs_num_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_8; 740 } else { 741 ena_trc_err(ena_dev, "Invalid descs_num_before_header, supported: 0x%x\n", 742 supported_feat); 743 return -EINVAL; 744 } 745 746 ena_trc_err(ena_dev, "Default llq num descs before header is not supported, performing fallback, default: 0x%x, supported: 0x%x, used: 0x%x\n", 747 llq_default_cfg->llq_num_decs_before_header, 748 supported_feat, 749 llq_info->descs_num_before_header); 750 } 751 /* Check for accelerated queue supported */ 752 llq_accel_mode_get = llq_features->accel_mode.u.get; 753 754 llq_info->disable_meta_caching = 755 !!(llq_accel_mode_get.supported_flags & 756 BIT(ENA_ADMIN_DISABLE_META_CACHING)); 757 758 if (llq_accel_mode_get.supported_flags & BIT(ENA_ADMIN_LIMIT_TX_BURST)) 759 llq_info->max_entries_in_tx_burst = 760 llq_accel_mode_get.max_tx_burst_size / 761 llq_default_cfg->llq_ring_entry_size_value; 762 763 rc = ena_com_set_llq(ena_dev); 764 if (rc) 765 ena_trc_err(ena_dev, "Cannot set LLQ configuration: %d\n", rc); 766 767 return rc; 768 } 769 770 static int ena_com_wait_and_process_admin_cq_interrupts(struct ena_comp_ctx *comp_ctx, 771 struct ena_com_admin_queue *admin_queue) 772 { 773 unsigned long flags = 0; 774 int ret; 775 776 ENA_WAIT_EVENT_WAIT(comp_ctx->wait_event, 777 admin_queue->completion_timeout); 778 779 /* In case the command wasn't completed find out the root cause. 780 * There might be 2 kinds of errors 781 * 1) No completion (timeout reached) 782 * 2) There is completion but the device didn't get any msi-x interrupt. 783 */ 784 if (unlikely(comp_ctx->status == ENA_CMD_SUBMITTED)) { 785 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); 786 ena_com_handle_admin_completion(admin_queue); 787 admin_queue->stats.no_completion++; 788 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); 789 790 if (comp_ctx->status == ENA_CMD_COMPLETED) { 791 ena_trc_err(admin_queue->ena_dev, 792 "The ena device sent a completion but the driver didn't receive a MSI-X interrupt (cmd %d), autopolling mode is %s\n", 793 comp_ctx->cmd_opcode, admin_queue->auto_polling ? "ON" : "OFF"); 794 /* Check if fallback to polling is enabled */ 795 if (admin_queue->auto_polling) 796 admin_queue->polling = true; 797 } else { 798 ena_trc_err(admin_queue->ena_dev, 799 "The ena device didn't send a completion for the admin cmd %d status %d\n", 800 comp_ctx->cmd_opcode, comp_ctx->status); 801 } 802 /* Check if shifted to polling mode. 803 * This will happen if there is a completion without an interrupt 804 * and autopolling mode is enabled. Continuing normal execution in such case 805 */ 806 if (!admin_queue->polling) { 807 admin_queue->running_state = false; 808 ret = ENA_COM_TIMER_EXPIRED; 809 goto err; 810 } 811 } 812 813 ret = ena_com_comp_status_to_errno(admin_queue, comp_ctx->comp_status); 814 err: 815 comp_ctxt_release(admin_queue, comp_ctx); 816 return ret; 817 } 818 819 /* This method read the hardware device register through posting writes 820 * and waiting for response 821 * On timeout the function will return ENA_MMIO_READ_TIMEOUT 822 */ 823 static u32 ena_com_reg_bar_read32(struct ena_com_dev *ena_dev, u16 offset) 824 { 825 struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read; 826 volatile struct ena_admin_ena_mmio_req_read_less_resp *read_resp = 827 mmio_read->read_resp; 828 u32 mmio_read_reg, ret, i; 829 unsigned long flags = 0; 830 u32 timeout = mmio_read->reg_read_to; 831 832 ENA_MIGHT_SLEEP(); 833 834 if (timeout == 0) 835 timeout = ENA_REG_READ_TIMEOUT; 836 837 /* If readless is disabled, perform regular read */ 838 if (!mmio_read->readless_supported) 839 return ENA_REG_READ32(ena_dev->bus, ena_dev->reg_bar + offset); 840 841 ENA_SPINLOCK_LOCK(mmio_read->lock, flags); 842 mmio_read->seq_num++; 843 844 read_resp->req_id = mmio_read->seq_num + 0xDEAD; 845 mmio_read_reg = (offset << ENA_REGS_MMIO_REG_READ_REG_OFF_SHIFT) & 846 ENA_REGS_MMIO_REG_READ_REG_OFF_MASK; 847 mmio_read_reg |= mmio_read->seq_num & 848 ENA_REGS_MMIO_REG_READ_REQ_ID_MASK; 849 850 ENA_REG_WRITE32(ena_dev->bus, mmio_read_reg, 851 ena_dev->reg_bar + ENA_REGS_MMIO_REG_READ_OFF); 852 853 for (i = 0; i < timeout; i++) { 854 if (READ_ONCE16(read_resp->req_id) == mmio_read->seq_num) 855 break; 856 857 ENA_UDELAY(1); 858 } 859 860 if (unlikely(i == timeout)) { 861 ena_trc_err(ena_dev, "Reading reg failed for timeout. expected: req id[%hu] offset[%hu] actual: req id[%hu] offset[%hu]\n", 862 mmio_read->seq_num, 863 offset, 864 read_resp->req_id, 865 read_resp->reg_off); 866 ret = ENA_MMIO_READ_TIMEOUT; 867 goto err; 868 } 869 870 if (read_resp->reg_off != offset) { 871 ena_trc_err(ena_dev, "Read failure: wrong offset provided\n"); 872 ret = ENA_MMIO_READ_TIMEOUT; 873 } else { 874 ret = read_resp->reg_val; 875 } 876 err: 877 ENA_SPINLOCK_UNLOCK(mmio_read->lock, flags); 878 879 return ret; 880 } 881 882 /* There are two types to wait for completion. 883 * Polling mode - wait until the completion is available. 884 * Async mode - wait on wait queue until the completion is ready 885 * (or the timeout expired). 886 * It is expected that the IRQ called ena_com_handle_admin_completion 887 * to mark the completions. 888 */ 889 static int ena_com_wait_and_process_admin_cq(struct ena_comp_ctx *comp_ctx, 890 struct ena_com_admin_queue *admin_queue) 891 { 892 if (admin_queue->polling) 893 return ena_com_wait_and_process_admin_cq_polling(comp_ctx, 894 admin_queue); 895 896 return ena_com_wait_and_process_admin_cq_interrupts(comp_ctx, 897 admin_queue); 898 } 899 900 static int ena_com_destroy_io_sq(struct ena_com_dev *ena_dev, 901 struct ena_com_io_sq *io_sq) 902 { 903 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; 904 struct ena_admin_aq_destroy_sq_cmd destroy_cmd; 905 struct ena_admin_acq_destroy_sq_resp_desc destroy_resp; 906 u8 direction; 907 int ret; 908 909 memset(&destroy_cmd, 0x0, sizeof(destroy_cmd)); 910 911 if (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) 912 direction = ENA_ADMIN_SQ_DIRECTION_TX; 913 else 914 direction = ENA_ADMIN_SQ_DIRECTION_RX; 915 916 destroy_cmd.sq.sq_identity |= (direction << 917 ENA_ADMIN_SQ_SQ_DIRECTION_SHIFT) & 918 ENA_ADMIN_SQ_SQ_DIRECTION_MASK; 919 920 destroy_cmd.sq.sq_idx = io_sq->idx; 921 destroy_cmd.aq_common_descriptor.opcode = ENA_ADMIN_DESTROY_SQ; 922 923 ret = ena_com_execute_admin_command(admin_queue, 924 (struct ena_admin_aq_entry *)&destroy_cmd, 925 sizeof(destroy_cmd), 926 (struct ena_admin_acq_entry *)&destroy_resp, 927 sizeof(destroy_resp)); 928 929 if (unlikely(ret && (ret != ENA_COM_NO_DEVICE))) 930 ena_trc_err(ena_dev, "Failed to destroy io sq error: %d\n", ret); 931 932 return ret; 933 } 934 935 static void ena_com_io_queue_free(struct ena_com_dev *ena_dev, 936 struct ena_com_io_sq *io_sq, 937 struct ena_com_io_cq *io_cq) 938 { 939 size_t size; 940 941 if (io_cq->cdesc_addr.virt_addr) { 942 size = io_cq->cdesc_entry_size_in_bytes * io_cq->q_depth; 943 944 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, 945 size, 946 io_cq->cdesc_addr.virt_addr, 947 io_cq->cdesc_addr.phys_addr, 948 io_cq->cdesc_addr.mem_handle); 949 950 io_cq->cdesc_addr.virt_addr = NULL; 951 } 952 953 if (io_sq->desc_addr.virt_addr) { 954 size = io_sq->desc_entry_size * io_sq->q_depth; 955 956 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, 957 size, 958 io_sq->desc_addr.virt_addr, 959 io_sq->desc_addr.phys_addr, 960 io_sq->desc_addr.mem_handle); 961 962 io_sq->desc_addr.virt_addr = NULL; 963 } 964 965 if (io_sq->bounce_buf_ctrl.base_buffer) { 966 ENA_MEM_FREE(ena_dev->dmadev, 967 io_sq->bounce_buf_ctrl.base_buffer, 968 (io_sq->llq_info.desc_list_entry_size * ENA_COM_BOUNCE_BUFFER_CNTRL_CNT)); 969 io_sq->bounce_buf_ctrl.base_buffer = NULL; 970 } 971 } 972 973 static int wait_for_reset_state(struct ena_com_dev *ena_dev, u32 timeout, 974 u16 exp_state) 975 { 976 u32 val, exp = 0; 977 ena_time_t timeout_stamp; 978 979 /* Convert timeout from resolution of 100ms to us resolution. */ 980 timeout_stamp = ENA_GET_SYSTEM_TIMEOUT(100 * 1000 * timeout); 981 982 while (1) { 983 val = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF); 984 985 if (unlikely(val == ENA_MMIO_READ_TIMEOUT)) { 986 ena_trc_err(ena_dev, "Reg read timeout occurred\n"); 987 return ENA_COM_TIMER_EXPIRED; 988 } 989 990 if ((val & ENA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK) == 991 exp_state) 992 return 0; 993 994 if (ENA_TIME_EXPIRE(timeout_stamp)) 995 return ENA_COM_TIMER_EXPIRED; 996 997 ena_delay_exponential_backoff_us(exp++, ena_dev->ena_min_poll_delay_us); 998 } 999 } 1000 1001 static bool ena_com_check_supported_feature_id(struct ena_com_dev *ena_dev, 1002 enum ena_admin_aq_feature_id feature_id) 1003 { 1004 u32 feature_mask = 1 << feature_id; 1005 1006 /* Device attributes is always supported */ 1007 if ((feature_id != ENA_ADMIN_DEVICE_ATTRIBUTES) && 1008 !(ena_dev->supported_features & feature_mask)) 1009 return false; 1010 1011 return true; 1012 } 1013 1014 static int ena_com_get_feature_ex(struct ena_com_dev *ena_dev, 1015 struct ena_admin_get_feat_resp *get_resp, 1016 enum ena_admin_aq_feature_id feature_id, 1017 dma_addr_t control_buf_dma_addr, 1018 u32 control_buff_size, 1019 u8 feature_ver) 1020 { 1021 struct ena_com_admin_queue *admin_queue; 1022 struct ena_admin_get_feat_cmd get_cmd; 1023 int ret; 1024 1025 if (!ena_com_check_supported_feature_id(ena_dev, feature_id)) { 1026 ena_trc_dbg(ena_dev, "Feature %d isn't supported\n", feature_id); 1027 return ENA_COM_UNSUPPORTED; 1028 } 1029 1030 memset(&get_cmd, 0x0, sizeof(get_cmd)); 1031 admin_queue = &ena_dev->admin_queue; 1032 1033 get_cmd.aq_common_descriptor.opcode = ENA_ADMIN_GET_FEATURE; 1034 1035 if (control_buff_size) 1036 get_cmd.aq_common_descriptor.flags = 1037 ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK; 1038 else 1039 get_cmd.aq_common_descriptor.flags = 0; 1040 1041 ret = ena_com_mem_addr_set(ena_dev, 1042 &get_cmd.control_buffer.address, 1043 control_buf_dma_addr); 1044 if (unlikely(ret)) { 1045 ena_trc_err(ena_dev, "Memory address set failed\n"); 1046 return ret; 1047 } 1048 1049 get_cmd.control_buffer.length = control_buff_size; 1050 get_cmd.feat_common.feature_version = feature_ver; 1051 get_cmd.feat_common.feature_id = feature_id; 1052 1053 ret = ena_com_execute_admin_command(admin_queue, 1054 (struct ena_admin_aq_entry *) 1055 &get_cmd, 1056 sizeof(get_cmd), 1057 (struct ena_admin_acq_entry *) 1058 get_resp, 1059 sizeof(*get_resp)); 1060 1061 if (unlikely(ret)) 1062 ena_trc_err(ena_dev, "Failed to submit get_feature command %d error: %d\n", 1063 feature_id, ret); 1064 1065 return ret; 1066 } 1067 1068 static int ena_com_get_feature(struct ena_com_dev *ena_dev, 1069 struct ena_admin_get_feat_resp *get_resp, 1070 enum ena_admin_aq_feature_id feature_id, 1071 u8 feature_ver) 1072 { 1073 return ena_com_get_feature_ex(ena_dev, 1074 get_resp, 1075 feature_id, 1076 0, 1077 0, 1078 feature_ver); 1079 } 1080 1081 int ena_com_get_current_hash_function(struct ena_com_dev *ena_dev) 1082 { 1083 return ena_dev->rss.hash_func; 1084 } 1085 1086 static void ena_com_hash_key_fill_default_key(struct ena_com_dev *ena_dev) 1087 { 1088 struct ena_admin_feature_rss_flow_hash_control *hash_key = 1089 (ena_dev->rss).hash_key; 1090 1091 ENA_RSS_FILL_KEY(&hash_key->key, sizeof(hash_key->key)); 1092 /* The key buffer is stored in the device in an array of 1093 * uint32 elements. 1094 */ 1095 hash_key->key_parts = ENA_ADMIN_RSS_KEY_PARTS; 1096 } 1097 1098 static int ena_com_hash_key_allocate(struct ena_com_dev *ena_dev) 1099 { 1100 struct ena_rss *rss = &ena_dev->rss; 1101 1102 if (!ena_com_check_supported_feature_id(ena_dev, ENA_ADMIN_RSS_HASH_FUNCTION)) 1103 return ENA_COM_UNSUPPORTED; 1104 1105 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, 1106 sizeof(*rss->hash_key), 1107 rss->hash_key, 1108 rss->hash_key_dma_addr, 1109 rss->hash_key_mem_handle); 1110 1111 if (unlikely(!rss->hash_key)) 1112 return ENA_COM_NO_MEM; 1113 1114 return 0; 1115 } 1116 1117 static void ena_com_hash_key_destroy(struct ena_com_dev *ena_dev) 1118 { 1119 struct ena_rss *rss = &ena_dev->rss; 1120 1121 if (rss->hash_key) 1122 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, 1123 sizeof(*rss->hash_key), 1124 rss->hash_key, 1125 rss->hash_key_dma_addr, 1126 rss->hash_key_mem_handle); 1127 rss->hash_key = NULL; 1128 } 1129 1130 static int ena_com_hash_ctrl_init(struct ena_com_dev *ena_dev) 1131 { 1132 struct ena_rss *rss = &ena_dev->rss; 1133 1134 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, 1135 sizeof(*rss->hash_ctrl), 1136 rss->hash_ctrl, 1137 rss->hash_ctrl_dma_addr, 1138 rss->hash_ctrl_mem_handle); 1139 1140 if (unlikely(!rss->hash_ctrl)) 1141 return ENA_COM_NO_MEM; 1142 1143 return 0; 1144 } 1145 1146 static void ena_com_hash_ctrl_destroy(struct ena_com_dev *ena_dev) 1147 { 1148 struct ena_rss *rss = &ena_dev->rss; 1149 1150 if (rss->hash_ctrl) 1151 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, 1152 sizeof(*rss->hash_ctrl), 1153 rss->hash_ctrl, 1154 rss->hash_ctrl_dma_addr, 1155 rss->hash_ctrl_mem_handle); 1156 rss->hash_ctrl = NULL; 1157 } 1158 1159 static int ena_com_indirect_table_allocate(struct ena_com_dev *ena_dev, 1160 u16 log_size) 1161 { 1162 struct ena_rss *rss = &ena_dev->rss; 1163 struct ena_admin_get_feat_resp get_resp; 1164 size_t tbl_size; 1165 int ret; 1166 1167 ret = ena_com_get_feature(ena_dev, &get_resp, 1168 ENA_ADMIN_RSS_INDIRECTION_TABLE_CONFIG, 0); 1169 if (unlikely(ret)) 1170 return ret; 1171 1172 if ((get_resp.u.ind_table.min_size > log_size) || 1173 (get_resp.u.ind_table.max_size < log_size)) { 1174 ena_trc_err(ena_dev, "Indirect table size doesn't fit. requested size: %d while min is:%d and max %d\n", 1175 1 << log_size, 1176 1 << get_resp.u.ind_table.min_size, 1177 1 << get_resp.u.ind_table.max_size); 1178 return ENA_COM_INVAL; 1179 } 1180 1181 tbl_size = (1ULL << log_size) * 1182 sizeof(struct ena_admin_rss_ind_table_entry); 1183 1184 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, 1185 tbl_size, 1186 rss->rss_ind_tbl, 1187 rss->rss_ind_tbl_dma_addr, 1188 rss->rss_ind_tbl_mem_handle); 1189 if (unlikely(!rss->rss_ind_tbl)) 1190 goto mem_err1; 1191 1192 tbl_size = (1ULL << log_size) * sizeof(u16); 1193 rss->host_rss_ind_tbl = 1194 ENA_MEM_ALLOC(ena_dev->dmadev, tbl_size); 1195 if (unlikely(!rss->host_rss_ind_tbl)) 1196 goto mem_err2; 1197 1198 rss->tbl_log_size = log_size; 1199 1200 return 0; 1201 1202 mem_err2: 1203 tbl_size = (1ULL << log_size) * 1204 sizeof(struct ena_admin_rss_ind_table_entry); 1205 1206 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, 1207 tbl_size, 1208 rss->rss_ind_tbl, 1209 rss->rss_ind_tbl_dma_addr, 1210 rss->rss_ind_tbl_mem_handle); 1211 rss->rss_ind_tbl = NULL; 1212 mem_err1: 1213 rss->tbl_log_size = 0; 1214 return ENA_COM_NO_MEM; 1215 } 1216 1217 static void ena_com_indirect_table_destroy(struct ena_com_dev *ena_dev) 1218 { 1219 struct ena_rss *rss = &ena_dev->rss; 1220 size_t tbl_size = (1ULL << rss->tbl_log_size) * 1221 sizeof(struct ena_admin_rss_ind_table_entry); 1222 1223 if (rss->rss_ind_tbl) 1224 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, 1225 tbl_size, 1226 rss->rss_ind_tbl, 1227 rss->rss_ind_tbl_dma_addr, 1228 rss->rss_ind_tbl_mem_handle); 1229 rss->rss_ind_tbl = NULL; 1230 1231 if (rss->host_rss_ind_tbl) 1232 ENA_MEM_FREE(ena_dev->dmadev, 1233 rss->host_rss_ind_tbl, 1234 ((1ULL << rss->tbl_log_size) * sizeof(u16))); 1235 rss->host_rss_ind_tbl = NULL; 1236 } 1237 1238 static int ena_com_create_io_sq(struct ena_com_dev *ena_dev, 1239 struct ena_com_io_sq *io_sq, u16 cq_idx) 1240 { 1241 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; 1242 struct ena_admin_aq_create_sq_cmd create_cmd; 1243 struct ena_admin_acq_create_sq_resp_desc cmd_completion; 1244 u8 direction; 1245 int ret; 1246 1247 memset(&create_cmd, 0x0, sizeof(create_cmd)); 1248 1249 create_cmd.aq_common_descriptor.opcode = ENA_ADMIN_CREATE_SQ; 1250 1251 if (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) 1252 direction = ENA_ADMIN_SQ_DIRECTION_TX; 1253 else 1254 direction = ENA_ADMIN_SQ_DIRECTION_RX; 1255 1256 create_cmd.sq_identity |= (direction << 1257 ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_SHIFT) & 1258 ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_MASK; 1259 1260 create_cmd.sq_caps_2 |= io_sq->mem_queue_type & 1261 ENA_ADMIN_AQ_CREATE_SQ_CMD_PLACEMENT_POLICY_MASK; 1262 1263 create_cmd.sq_caps_2 |= (ENA_ADMIN_COMPLETION_POLICY_DESC << 1264 ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_SHIFT) & 1265 ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_MASK; 1266 1267 create_cmd.sq_caps_3 |= 1268 ENA_ADMIN_AQ_CREATE_SQ_CMD_IS_PHYSICALLY_CONTIGUOUS_MASK; 1269 1270 create_cmd.cq_idx = cq_idx; 1271 create_cmd.sq_depth = io_sq->q_depth; 1272 1273 if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) { 1274 ret = ena_com_mem_addr_set(ena_dev, 1275 &create_cmd.sq_ba, 1276 io_sq->desc_addr.phys_addr); 1277 if (unlikely(ret)) { 1278 ena_trc_err(ena_dev, "Memory address set failed\n"); 1279 return ret; 1280 } 1281 } 1282 1283 ret = ena_com_execute_admin_command(admin_queue, 1284 (struct ena_admin_aq_entry *)&create_cmd, 1285 sizeof(create_cmd), 1286 (struct ena_admin_acq_entry *)&cmd_completion, 1287 sizeof(cmd_completion)); 1288 if (unlikely(ret)) { 1289 ena_trc_err(ena_dev, "Failed to create IO SQ. error: %d\n", ret); 1290 return ret; 1291 } 1292 1293 io_sq->idx = cmd_completion.sq_idx; 1294 1295 io_sq->db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar + 1296 (uintptr_t)cmd_completion.sq_doorbell_offset); 1297 1298 if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { 1299 io_sq->header_addr = (u8 __iomem *)((uintptr_t)ena_dev->mem_bar 1300 + cmd_completion.llq_headers_offset); 1301 1302 io_sq->desc_addr.pbuf_dev_addr = 1303 (u8 __iomem *)((uintptr_t)ena_dev->mem_bar + 1304 cmd_completion.llq_descriptors_offset); 1305 } 1306 1307 ena_trc_dbg(ena_dev, "Created sq[%u], depth[%u]\n", io_sq->idx, io_sq->q_depth); 1308 1309 return ret; 1310 } 1311 1312 static int ena_com_ind_tbl_convert_to_device(struct ena_com_dev *ena_dev) 1313 { 1314 struct ena_rss *rss = &ena_dev->rss; 1315 struct ena_com_io_sq *io_sq; 1316 u16 qid; 1317 int i; 1318 1319 for (i = 0; i < 1 << rss->tbl_log_size; i++) { 1320 qid = rss->host_rss_ind_tbl[i]; 1321 if (qid >= ENA_TOTAL_NUM_QUEUES) 1322 return ENA_COM_INVAL; 1323 1324 io_sq = &ena_dev->io_sq_queues[qid]; 1325 1326 if (io_sq->direction != ENA_COM_IO_QUEUE_DIRECTION_RX) 1327 return ENA_COM_INVAL; 1328 1329 rss->rss_ind_tbl[i].cq_idx = io_sq->idx; 1330 } 1331 1332 return 0; 1333 } 1334 1335 static void ena_com_update_intr_delay_resolution(struct ena_com_dev *ena_dev, 1336 u16 intr_delay_resolution) 1337 { 1338 u16 prev_intr_delay_resolution = ena_dev->intr_delay_resolution; 1339 1340 if (unlikely(!intr_delay_resolution)) { 1341 ena_trc_err(ena_dev, "Illegal intr_delay_resolution provided. Going to use default 1 usec resolution\n"); 1342 intr_delay_resolution = ENA_DEFAULT_INTR_DELAY_RESOLUTION; 1343 } 1344 1345 /* update Rx */ 1346 ena_dev->intr_moder_rx_interval = 1347 ena_dev->intr_moder_rx_interval * 1348 prev_intr_delay_resolution / 1349 intr_delay_resolution; 1350 1351 /* update Tx */ 1352 ena_dev->intr_moder_tx_interval = 1353 ena_dev->intr_moder_tx_interval * 1354 prev_intr_delay_resolution / 1355 intr_delay_resolution; 1356 1357 ena_dev->intr_delay_resolution = intr_delay_resolution; 1358 } 1359 1360 /*****************************************************************************/ 1361 /******************************* API ******************************/ 1362 /*****************************************************************************/ 1363 1364 int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue, 1365 struct ena_admin_aq_entry *cmd, 1366 size_t cmd_size, 1367 struct ena_admin_acq_entry *comp, 1368 size_t comp_size) 1369 { 1370 struct ena_comp_ctx *comp_ctx; 1371 int ret; 1372 1373 comp_ctx = ena_com_submit_admin_cmd(admin_queue, cmd, cmd_size, 1374 comp, comp_size); 1375 if (IS_ERR(comp_ctx)) { 1376 if (comp_ctx == ERR_PTR(ENA_COM_NO_DEVICE)) 1377 ena_trc_dbg(admin_queue->ena_dev, 1378 "Failed to submit command [%ld]\n", 1379 PTR_ERR(comp_ctx)); 1380 else 1381 ena_trc_err(admin_queue->ena_dev, 1382 "Failed to submit command [%ld]\n", 1383 PTR_ERR(comp_ctx)); 1384 1385 return (int)PTR_ERR(comp_ctx); 1386 } 1387 1388 ret = ena_com_wait_and_process_admin_cq(comp_ctx, admin_queue); 1389 if (unlikely(ret)) { 1390 if (admin_queue->running_state) 1391 ena_trc_err(admin_queue->ena_dev, 1392 "Failed to process command. ret = %d\n", ret); 1393 else 1394 ena_trc_dbg(admin_queue->ena_dev, 1395 "Failed to process command. ret = %d\n", ret); 1396 } 1397 return ret; 1398 } 1399 1400 int ena_com_create_io_cq(struct ena_com_dev *ena_dev, 1401 struct ena_com_io_cq *io_cq) 1402 { 1403 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; 1404 struct ena_admin_aq_create_cq_cmd create_cmd; 1405 struct ena_admin_acq_create_cq_resp_desc cmd_completion; 1406 int ret; 1407 1408 memset(&create_cmd, 0x0, sizeof(create_cmd)); 1409 1410 create_cmd.aq_common_descriptor.opcode = ENA_ADMIN_CREATE_CQ; 1411 1412 create_cmd.cq_caps_2 |= (io_cq->cdesc_entry_size_in_bytes / 4) & 1413 ENA_ADMIN_AQ_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK; 1414 create_cmd.cq_caps_1 |= 1415 ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_MASK; 1416 1417 create_cmd.msix_vector = io_cq->msix_vector; 1418 create_cmd.cq_depth = io_cq->q_depth; 1419 1420 ret = ena_com_mem_addr_set(ena_dev, 1421 &create_cmd.cq_ba, 1422 io_cq->cdesc_addr.phys_addr); 1423 if (unlikely(ret)) { 1424 ena_trc_err(ena_dev, "Memory address set failed\n"); 1425 return ret; 1426 } 1427 1428 ret = ena_com_execute_admin_command(admin_queue, 1429 (struct ena_admin_aq_entry *)&create_cmd, 1430 sizeof(create_cmd), 1431 (struct ena_admin_acq_entry *)&cmd_completion, 1432 sizeof(cmd_completion)); 1433 if (unlikely(ret)) { 1434 ena_trc_err(ena_dev, "Failed to create IO CQ. error: %d\n", ret); 1435 return ret; 1436 } 1437 1438 io_cq->idx = cmd_completion.cq_idx; 1439 1440 io_cq->unmask_reg = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar + 1441 cmd_completion.cq_interrupt_unmask_register_offset); 1442 1443 if (cmd_completion.cq_head_db_register_offset) 1444 io_cq->cq_head_db_reg = 1445 (u32 __iomem *)((uintptr_t)ena_dev->reg_bar + 1446 cmd_completion.cq_head_db_register_offset); 1447 1448 if (cmd_completion.numa_node_register_offset) 1449 io_cq->numa_node_cfg_reg = 1450 (u32 __iomem *)((uintptr_t)ena_dev->reg_bar + 1451 cmd_completion.numa_node_register_offset); 1452 1453 ena_trc_dbg(ena_dev, "Created cq[%u], depth[%u]\n", io_cq->idx, io_cq->q_depth); 1454 1455 return ret; 1456 } 1457 1458 int ena_com_get_io_handlers(struct ena_com_dev *ena_dev, u16 qid, 1459 struct ena_com_io_sq **io_sq, 1460 struct ena_com_io_cq **io_cq) 1461 { 1462 if (qid >= ENA_TOTAL_NUM_QUEUES) { 1463 ena_trc_err(ena_dev, "Invalid queue number %d but the max is %d\n", 1464 qid, ENA_TOTAL_NUM_QUEUES); 1465 return ENA_COM_INVAL; 1466 } 1467 1468 *io_sq = &ena_dev->io_sq_queues[qid]; 1469 *io_cq = &ena_dev->io_cq_queues[qid]; 1470 1471 return 0; 1472 } 1473 1474 void ena_com_abort_admin_commands(struct ena_com_dev *ena_dev) 1475 { 1476 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; 1477 struct ena_comp_ctx *comp_ctx; 1478 u16 i; 1479 1480 if (!admin_queue->comp_ctx) 1481 return; 1482 1483 for (i = 0; i < admin_queue->q_depth; i++) { 1484 comp_ctx = get_comp_ctxt(admin_queue, i, false); 1485 if (unlikely(!comp_ctx)) 1486 break; 1487 1488 comp_ctx->status = ENA_CMD_ABORTED; 1489 1490 ENA_WAIT_EVENT_SIGNAL(comp_ctx->wait_event); 1491 } 1492 } 1493 1494 void ena_com_wait_for_abort_completion(struct ena_com_dev *ena_dev) 1495 { 1496 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; 1497 unsigned long flags = 0; 1498 u32 exp = 0; 1499 1500 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); 1501 while (ATOMIC32_READ(&admin_queue->outstanding_cmds) != 0) { 1502 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); 1503 ena_delay_exponential_backoff_us(exp++, ena_dev->ena_min_poll_delay_us); 1504 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); 1505 } 1506 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); 1507 } 1508 1509 int ena_com_destroy_io_cq(struct ena_com_dev *ena_dev, 1510 struct ena_com_io_cq *io_cq) 1511 { 1512 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; 1513 struct ena_admin_aq_destroy_cq_cmd destroy_cmd; 1514 struct ena_admin_acq_destroy_cq_resp_desc destroy_resp; 1515 int ret; 1516 1517 memset(&destroy_cmd, 0x0, sizeof(destroy_cmd)); 1518 1519 destroy_cmd.cq_idx = io_cq->idx; 1520 destroy_cmd.aq_common_descriptor.opcode = ENA_ADMIN_DESTROY_CQ; 1521 1522 ret = ena_com_execute_admin_command(admin_queue, 1523 (struct ena_admin_aq_entry *)&destroy_cmd, 1524 sizeof(destroy_cmd), 1525 (struct ena_admin_acq_entry *)&destroy_resp, 1526 sizeof(destroy_resp)); 1527 1528 if (unlikely(ret && (ret != ENA_COM_NO_DEVICE))) 1529 ena_trc_err(ena_dev, "Failed to destroy IO CQ. error: %d\n", ret); 1530 1531 return ret; 1532 } 1533 1534 bool ena_com_get_admin_running_state(struct ena_com_dev *ena_dev) 1535 { 1536 return ena_dev->admin_queue.running_state; 1537 } 1538 1539 void ena_com_set_admin_running_state(struct ena_com_dev *ena_dev, bool state) 1540 { 1541 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; 1542 unsigned long flags = 0; 1543 1544 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); 1545 ena_dev->admin_queue.running_state = state; 1546 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); 1547 } 1548 1549 void ena_com_admin_aenq_enable(struct ena_com_dev *ena_dev) 1550 { 1551 u16 depth = ena_dev->aenq.q_depth; 1552 1553 ENA_WARN(ena_dev->aenq.head != depth, ena_dev, "Invalid AENQ state\n"); 1554 1555 /* Init head_db to mark that all entries in the queue 1556 * are initially available 1557 */ 1558 ENA_REG_WRITE32(ena_dev->bus, depth, ena_dev->reg_bar + ENA_REGS_AENQ_HEAD_DB_OFF); 1559 } 1560 1561 int ena_com_set_aenq_config(struct ena_com_dev *ena_dev, u32 groups_flag) 1562 { 1563 struct ena_com_admin_queue *admin_queue; 1564 struct ena_admin_set_feat_cmd cmd; 1565 struct ena_admin_set_feat_resp resp; 1566 struct ena_admin_get_feat_resp get_resp; 1567 int ret; 1568 1569 ret = ena_com_get_feature(ena_dev, &get_resp, ENA_ADMIN_AENQ_CONFIG, 0); 1570 if (ret) { 1571 ena_trc_info(ena_dev, "Can't get aenq configuration\n"); 1572 return ret; 1573 } 1574 1575 if ((get_resp.u.aenq.supported_groups & groups_flag) != groups_flag) { 1576 ena_trc_warn(ena_dev, "Trying to set unsupported aenq events. supported flag: 0x%x asked flag: 0x%x\n", 1577 get_resp.u.aenq.supported_groups, 1578 groups_flag); 1579 return ENA_COM_UNSUPPORTED; 1580 } 1581 1582 memset(&cmd, 0x0, sizeof(cmd)); 1583 admin_queue = &ena_dev->admin_queue; 1584 1585 cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE; 1586 cmd.aq_common_descriptor.flags = 0; 1587 cmd.feat_common.feature_id = ENA_ADMIN_AENQ_CONFIG; 1588 cmd.u.aenq.enabled_groups = groups_flag; 1589 1590 ret = ena_com_execute_admin_command(admin_queue, 1591 (struct ena_admin_aq_entry *)&cmd, 1592 sizeof(cmd), 1593 (struct ena_admin_acq_entry *)&resp, 1594 sizeof(resp)); 1595 1596 if (unlikely(ret)) 1597 ena_trc_err(ena_dev, "Failed to config AENQ ret: %d\n", ret); 1598 1599 return ret; 1600 } 1601 1602 int ena_com_get_dma_width(struct ena_com_dev *ena_dev) 1603 { 1604 u32 caps = ena_com_reg_bar_read32(ena_dev, ENA_REGS_CAPS_OFF); 1605 u32 width; 1606 1607 if (unlikely(caps == ENA_MMIO_READ_TIMEOUT)) { 1608 ena_trc_err(ena_dev, "Reg read timeout occurred\n"); 1609 return ENA_COM_TIMER_EXPIRED; 1610 } 1611 1612 width = (caps & ENA_REGS_CAPS_DMA_ADDR_WIDTH_MASK) >> 1613 ENA_REGS_CAPS_DMA_ADDR_WIDTH_SHIFT; 1614 1615 ena_trc_dbg(ena_dev, "ENA dma width: %d\n", width); 1616 1617 if ((width < 32) || width > ENA_MAX_PHYS_ADDR_SIZE_BITS) { 1618 ena_trc_err(ena_dev, "DMA width illegal value: %d\n", width); 1619 return ENA_COM_INVAL; 1620 } 1621 1622 ena_dev->dma_addr_bits = width; 1623 1624 return width; 1625 } 1626 1627 int ena_com_validate_version(struct ena_com_dev *ena_dev) 1628 { 1629 u32 ver; 1630 u32 ctrl_ver; 1631 u32 ctrl_ver_masked; 1632 1633 /* Make sure the ENA version and the controller version are at least 1634 * as the driver expects 1635 */ 1636 ver = ena_com_reg_bar_read32(ena_dev, ENA_REGS_VERSION_OFF); 1637 ctrl_ver = ena_com_reg_bar_read32(ena_dev, 1638 ENA_REGS_CONTROLLER_VERSION_OFF); 1639 1640 if (unlikely((ver == ENA_MMIO_READ_TIMEOUT) || 1641 (ctrl_ver == ENA_MMIO_READ_TIMEOUT))) { 1642 ena_trc_err(ena_dev, "Reg read timeout occurred\n"); 1643 return ENA_COM_TIMER_EXPIRED; 1644 } 1645 1646 ena_trc_info(ena_dev, "ENA device version: %d.%d\n", 1647 (ver & ENA_REGS_VERSION_MAJOR_VERSION_MASK) >> 1648 ENA_REGS_VERSION_MAJOR_VERSION_SHIFT, 1649 ver & ENA_REGS_VERSION_MINOR_VERSION_MASK); 1650 1651 ena_trc_info(ena_dev, "ENA controller version: %d.%d.%d implementation version %d\n", 1652 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK) 1653 >> ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT, 1654 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK) 1655 >> ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT, 1656 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK), 1657 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_IMPL_ID_MASK) >> 1658 ENA_REGS_CONTROLLER_VERSION_IMPL_ID_SHIFT); 1659 1660 ctrl_ver_masked = 1661 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK) | 1662 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK) | 1663 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK); 1664 1665 /* Validate the ctrl version without the implementation ID */ 1666 if (ctrl_ver_masked < MIN_ENA_CTRL_VER) { 1667 ena_trc_err(ena_dev, "ENA ctrl version is lower than the minimal ctrl version the driver supports\n"); 1668 return -1; 1669 } 1670 1671 return 0; 1672 } 1673 1674 static void 1675 ena_com_free_ena_admin_queue_comp_ctx(struct ena_com_dev *ena_dev, 1676 struct ena_com_admin_queue *admin_queue) 1677 1678 { 1679 if (!admin_queue->comp_ctx) 1680 return; 1681 1682 ENA_WAIT_EVENTS_DESTROY(admin_queue); 1683 ENA_MEM_FREE(ena_dev->dmadev, 1684 admin_queue->comp_ctx, 1685 (admin_queue->q_depth * sizeof(struct ena_comp_ctx))); 1686 1687 admin_queue->comp_ctx = NULL; 1688 } 1689 1690 void ena_com_admin_destroy(struct ena_com_dev *ena_dev) 1691 { 1692 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; 1693 struct ena_com_admin_cq *cq = &admin_queue->cq; 1694 struct ena_com_admin_sq *sq = &admin_queue->sq; 1695 struct ena_com_aenq *aenq = &ena_dev->aenq; 1696 u16 size; 1697 1698 ena_com_free_ena_admin_queue_comp_ctx(ena_dev, admin_queue); 1699 1700 size = ADMIN_SQ_SIZE(admin_queue->q_depth); 1701 if (sq->entries) 1702 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, size, sq->entries, 1703 sq->dma_addr, sq->mem_handle); 1704 sq->entries = NULL; 1705 1706 size = ADMIN_CQ_SIZE(admin_queue->q_depth); 1707 if (cq->entries) 1708 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, size, cq->entries, 1709 cq->dma_addr, cq->mem_handle); 1710 cq->entries = NULL; 1711 1712 size = ADMIN_AENQ_SIZE(aenq->q_depth); 1713 if (ena_dev->aenq.entries) 1714 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, size, aenq->entries, 1715 aenq->dma_addr, aenq->mem_handle); 1716 aenq->entries = NULL; 1717 ENA_SPINLOCK_DESTROY(admin_queue->q_lock); 1718 } 1719 1720 void ena_com_set_admin_polling_mode(struct ena_com_dev *ena_dev, bool polling) 1721 { 1722 u32 mask_value = 0; 1723 1724 if (polling) 1725 mask_value = ENA_REGS_ADMIN_INTR_MASK; 1726 1727 ENA_REG_WRITE32(ena_dev->bus, mask_value, 1728 ena_dev->reg_bar + ENA_REGS_INTR_MASK_OFF); 1729 ena_dev->admin_queue.polling = polling; 1730 } 1731 1732 bool ena_com_get_admin_polling_mode(struct ena_com_dev *ena_dev) 1733 { 1734 return ena_dev->admin_queue.polling; 1735 } 1736 1737 void ena_com_set_admin_auto_polling_mode(struct ena_com_dev *ena_dev, 1738 bool polling) 1739 { 1740 ena_dev->admin_queue.auto_polling = polling; 1741 } 1742 1743 int ena_com_mmio_reg_read_request_init(struct ena_com_dev *ena_dev) 1744 { 1745 struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read; 1746 1747 ENA_SPINLOCK_INIT(mmio_read->lock); 1748 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, 1749 sizeof(*mmio_read->read_resp), 1750 mmio_read->read_resp, 1751 mmio_read->read_resp_dma_addr, 1752 mmio_read->read_resp_mem_handle); 1753 if (unlikely(!mmio_read->read_resp)) 1754 goto err; 1755 1756 ena_com_mmio_reg_read_request_write_dev_addr(ena_dev); 1757 1758 mmio_read->read_resp->req_id = 0x0; 1759 mmio_read->seq_num = 0x0; 1760 mmio_read->readless_supported = true; 1761 1762 return 0; 1763 1764 err: 1765 ENA_SPINLOCK_DESTROY(mmio_read->lock); 1766 return ENA_COM_NO_MEM; 1767 } 1768 1769 void ena_com_set_mmio_read_mode(struct ena_com_dev *ena_dev, bool readless_supported) 1770 { 1771 struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read; 1772 1773 mmio_read->readless_supported = readless_supported; 1774 } 1775 1776 void ena_com_mmio_reg_read_request_destroy(struct ena_com_dev *ena_dev) 1777 { 1778 struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read; 1779 1780 ENA_REG_WRITE32(ena_dev->bus, 0x0, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_LO_OFF); 1781 ENA_REG_WRITE32(ena_dev->bus, 0x0, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_HI_OFF); 1782 1783 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, 1784 sizeof(*mmio_read->read_resp), 1785 mmio_read->read_resp, 1786 mmio_read->read_resp_dma_addr, 1787 mmio_read->read_resp_mem_handle); 1788 1789 mmio_read->read_resp = NULL; 1790 ENA_SPINLOCK_DESTROY(mmio_read->lock); 1791 } 1792 1793 void ena_com_mmio_reg_read_request_write_dev_addr(struct ena_com_dev *ena_dev) 1794 { 1795 struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read; 1796 u32 addr_low, addr_high; 1797 1798 addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(mmio_read->read_resp_dma_addr); 1799 addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(mmio_read->read_resp_dma_addr); 1800 1801 ENA_REG_WRITE32(ena_dev->bus, addr_low, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_LO_OFF); 1802 ENA_REG_WRITE32(ena_dev->bus, addr_high, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_HI_OFF); 1803 } 1804 1805 int ena_com_admin_init(struct ena_com_dev *ena_dev, 1806 struct ena_aenq_handlers *aenq_handlers) 1807 { 1808 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; 1809 u32 aq_caps, acq_caps, dev_sts, addr_low, addr_high; 1810 int ret; 1811 1812 dev_sts = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF); 1813 1814 if (unlikely(dev_sts == ENA_MMIO_READ_TIMEOUT)) { 1815 ena_trc_err(ena_dev, "Reg read timeout occurred\n"); 1816 return ENA_COM_TIMER_EXPIRED; 1817 } 1818 1819 if (!(dev_sts & ENA_REGS_DEV_STS_READY_MASK)) { 1820 ena_trc_err(ena_dev, "Device isn't ready, abort com init\n"); 1821 return ENA_COM_NO_DEVICE; 1822 } 1823 1824 admin_queue->q_depth = ENA_ADMIN_QUEUE_DEPTH; 1825 1826 admin_queue->bus = ena_dev->bus; 1827 admin_queue->q_dmadev = ena_dev->dmadev; 1828 admin_queue->polling = false; 1829 admin_queue->curr_cmd_id = 0; 1830 1831 ATOMIC32_SET(&admin_queue->outstanding_cmds, 0); 1832 1833 ENA_SPINLOCK_INIT(admin_queue->q_lock); 1834 1835 ret = ena_com_init_comp_ctxt(admin_queue); 1836 if (ret) 1837 goto error; 1838 1839 ret = ena_com_admin_init_sq(admin_queue); 1840 if (ret) 1841 goto error; 1842 1843 ret = ena_com_admin_init_cq(admin_queue); 1844 if (ret) 1845 goto error; 1846 1847 admin_queue->sq.db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar + 1848 ENA_REGS_AQ_DB_OFF); 1849 1850 addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->sq.dma_addr); 1851 addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->sq.dma_addr); 1852 1853 ENA_REG_WRITE32(ena_dev->bus, addr_low, ena_dev->reg_bar + ENA_REGS_AQ_BASE_LO_OFF); 1854 ENA_REG_WRITE32(ena_dev->bus, addr_high, ena_dev->reg_bar + ENA_REGS_AQ_BASE_HI_OFF); 1855 1856 addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->cq.dma_addr); 1857 addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->cq.dma_addr); 1858 1859 ENA_REG_WRITE32(ena_dev->bus, addr_low, ena_dev->reg_bar + ENA_REGS_ACQ_BASE_LO_OFF); 1860 ENA_REG_WRITE32(ena_dev->bus, addr_high, ena_dev->reg_bar + ENA_REGS_ACQ_BASE_HI_OFF); 1861 1862 aq_caps = 0; 1863 aq_caps |= admin_queue->q_depth & ENA_REGS_AQ_CAPS_AQ_DEPTH_MASK; 1864 aq_caps |= (sizeof(struct ena_admin_aq_entry) << 1865 ENA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_SHIFT) & 1866 ENA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_MASK; 1867 1868 acq_caps = 0; 1869 acq_caps |= admin_queue->q_depth & ENA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK; 1870 acq_caps |= (sizeof(struct ena_admin_acq_entry) << 1871 ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_SHIFT) & 1872 ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_MASK; 1873 1874 ENA_REG_WRITE32(ena_dev->bus, aq_caps, ena_dev->reg_bar + ENA_REGS_AQ_CAPS_OFF); 1875 ENA_REG_WRITE32(ena_dev->bus, acq_caps, ena_dev->reg_bar + ENA_REGS_ACQ_CAPS_OFF); 1876 ret = ena_com_admin_init_aenq(ena_dev, aenq_handlers); 1877 if (ret) 1878 goto error; 1879 1880 admin_queue->ena_dev = ena_dev; 1881 admin_queue->running_state = true; 1882 1883 return 0; 1884 error: 1885 ena_com_admin_destroy(ena_dev); 1886 1887 return ret; 1888 } 1889 1890 int ena_com_create_io_queue(struct ena_com_dev *ena_dev, 1891 struct ena_com_create_io_ctx *ctx) 1892 { 1893 struct ena_com_io_sq *io_sq; 1894 struct ena_com_io_cq *io_cq; 1895 int ret; 1896 1897 if (ctx->qid >= ENA_TOTAL_NUM_QUEUES) { 1898 ena_trc_err(ena_dev, "Qid (%d) is bigger than max num of queues (%d)\n", 1899 ctx->qid, ENA_TOTAL_NUM_QUEUES); 1900 return ENA_COM_INVAL; 1901 } 1902 1903 io_sq = &ena_dev->io_sq_queues[ctx->qid]; 1904 io_cq = &ena_dev->io_cq_queues[ctx->qid]; 1905 1906 memset(io_sq, 0x0, sizeof(*io_sq)); 1907 memset(io_cq, 0x0, sizeof(*io_cq)); 1908 1909 /* Init CQ */ 1910 io_cq->q_depth = ctx->queue_size; 1911 io_cq->direction = ctx->direction; 1912 io_cq->qid = ctx->qid; 1913 1914 io_cq->msix_vector = ctx->msix_vector; 1915 1916 io_sq->q_depth = ctx->queue_size; 1917 io_sq->direction = ctx->direction; 1918 io_sq->qid = ctx->qid; 1919 1920 io_sq->mem_queue_type = ctx->mem_queue_type; 1921 1922 if (ctx->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) 1923 /* header length is limited to 8 bits */ 1924 io_sq->tx_max_header_size = 1925 ENA_MIN32(ena_dev->tx_max_header_size, SZ_256); 1926 1927 ret = ena_com_init_io_sq(ena_dev, ctx, io_sq); 1928 if (ret) 1929 goto error; 1930 ret = ena_com_init_io_cq(ena_dev, ctx, io_cq); 1931 if (ret) 1932 goto error; 1933 1934 ret = ena_com_create_io_cq(ena_dev, io_cq); 1935 if (ret) 1936 goto error; 1937 1938 ret = ena_com_create_io_sq(ena_dev, io_sq, io_cq->idx); 1939 if (ret) 1940 goto destroy_io_cq; 1941 1942 return 0; 1943 1944 destroy_io_cq: 1945 ena_com_destroy_io_cq(ena_dev, io_cq); 1946 error: 1947 ena_com_io_queue_free(ena_dev, io_sq, io_cq); 1948 return ret; 1949 } 1950 1951 void ena_com_destroy_io_queue(struct ena_com_dev *ena_dev, u16 qid) 1952 { 1953 struct ena_com_io_sq *io_sq; 1954 struct ena_com_io_cq *io_cq; 1955 1956 if (qid >= ENA_TOTAL_NUM_QUEUES) { 1957 ena_trc_err(ena_dev, "Qid (%d) is bigger than max num of queues (%d)\n", 1958 qid, ENA_TOTAL_NUM_QUEUES); 1959 return; 1960 } 1961 1962 io_sq = &ena_dev->io_sq_queues[qid]; 1963 io_cq = &ena_dev->io_cq_queues[qid]; 1964 1965 ena_com_destroy_io_sq(ena_dev, io_sq); 1966 ena_com_destroy_io_cq(ena_dev, io_cq); 1967 1968 ena_com_io_queue_free(ena_dev, io_sq, io_cq); 1969 } 1970 1971 int ena_com_get_link_params(struct ena_com_dev *ena_dev, 1972 struct ena_admin_get_feat_resp *resp) 1973 { 1974 return ena_com_get_feature(ena_dev, resp, ENA_ADMIN_LINK_CONFIG, 0); 1975 } 1976 1977 int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev, 1978 struct ena_com_dev_get_features_ctx *get_feat_ctx) 1979 { 1980 struct ena_admin_get_feat_resp get_resp; 1981 int rc; 1982 1983 rc = ena_com_get_feature(ena_dev, &get_resp, 1984 ENA_ADMIN_DEVICE_ATTRIBUTES, 0); 1985 if (rc) 1986 return rc; 1987 1988 memcpy(&get_feat_ctx->dev_attr, &get_resp.u.dev_attr, 1989 sizeof(get_resp.u.dev_attr)); 1990 1991 ena_dev->supported_features = get_resp.u.dev_attr.supported_features; 1992 1993 if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) { 1994 rc = ena_com_get_feature(ena_dev, &get_resp, 1995 ENA_ADMIN_MAX_QUEUES_EXT, 1996 ENA_FEATURE_MAX_QUEUE_EXT_VER); 1997 if (rc) 1998 return rc; 1999 2000 if (get_resp.u.max_queue_ext.version != ENA_FEATURE_MAX_QUEUE_EXT_VER) 2001 return -EINVAL; 2002 2003 memcpy(&get_feat_ctx->max_queue_ext, &get_resp.u.max_queue_ext, 2004 sizeof(get_resp.u.max_queue_ext)); 2005 ena_dev->tx_max_header_size = 2006 get_resp.u.max_queue_ext.max_queue_ext.max_tx_header_size; 2007 } else { 2008 rc = ena_com_get_feature(ena_dev, &get_resp, 2009 ENA_ADMIN_MAX_QUEUES_NUM, 0); 2010 memcpy(&get_feat_ctx->max_queues, &get_resp.u.max_queue, 2011 sizeof(get_resp.u.max_queue)); 2012 ena_dev->tx_max_header_size = 2013 get_resp.u.max_queue.max_header_size; 2014 2015 if (rc) 2016 return rc; 2017 } 2018 2019 rc = ena_com_get_feature(ena_dev, &get_resp, 2020 ENA_ADMIN_AENQ_CONFIG, 0); 2021 if (rc) 2022 return rc; 2023 2024 memcpy(&get_feat_ctx->aenq, &get_resp.u.aenq, 2025 sizeof(get_resp.u.aenq)); 2026 2027 rc = ena_com_get_feature(ena_dev, &get_resp, 2028 ENA_ADMIN_STATELESS_OFFLOAD_CONFIG, 0); 2029 if (rc) 2030 return rc; 2031 2032 memcpy(&get_feat_ctx->offload, &get_resp.u.offload, 2033 sizeof(get_resp.u.offload)); 2034 2035 /* Driver hints isn't mandatory admin command. So in case the 2036 * command isn't supported set driver hints to 0 2037 */ 2038 rc = ena_com_get_feature(ena_dev, &get_resp, ENA_ADMIN_HW_HINTS, 0); 2039 2040 if (!rc) 2041 memcpy(&get_feat_ctx->hw_hints, &get_resp.u.hw_hints, 2042 sizeof(get_resp.u.hw_hints)); 2043 else if (rc == ENA_COM_UNSUPPORTED) 2044 memset(&get_feat_ctx->hw_hints, 0x0, sizeof(get_feat_ctx->hw_hints)); 2045 else 2046 return rc; 2047 2048 rc = ena_com_get_feature(ena_dev, &get_resp, ENA_ADMIN_LLQ, 0); 2049 if (!rc) 2050 memcpy(&get_feat_ctx->llq, &get_resp.u.llq, 2051 sizeof(get_resp.u.llq)); 2052 else if (rc == ENA_COM_UNSUPPORTED) 2053 memset(&get_feat_ctx->llq, 0x0, sizeof(get_feat_ctx->llq)); 2054 else 2055 return rc; 2056 2057 return 0; 2058 } 2059 2060 void ena_com_admin_q_comp_intr_handler(struct ena_com_dev *ena_dev) 2061 { 2062 ena_com_handle_admin_completion(&ena_dev->admin_queue); 2063 } 2064 2065 /* ena_handle_specific_aenq_event: 2066 * return the handler that is relevant to the specific event group 2067 */ 2068 static ena_aenq_handler ena_com_get_specific_aenq_cb(struct ena_com_dev *ena_dev, 2069 u16 group) 2070 { 2071 struct ena_aenq_handlers *aenq_handlers = ena_dev->aenq.aenq_handlers; 2072 2073 if ((group < ENA_MAX_HANDLERS) && aenq_handlers->handlers[group]) 2074 return aenq_handlers->handlers[group]; 2075 2076 return aenq_handlers->unimplemented_handler; 2077 } 2078 2079 /* ena_aenq_intr_handler: 2080 * handles the aenq incoming events. 2081 * pop events from the queue and apply the specific handler 2082 */ 2083 void ena_com_aenq_intr_handler(struct ena_com_dev *ena_dev, void *data) 2084 { 2085 struct ena_admin_aenq_entry *aenq_e; 2086 struct ena_admin_aenq_common_desc *aenq_common; 2087 struct ena_com_aenq *aenq = &ena_dev->aenq; 2088 u64 timestamp; 2089 ena_aenq_handler handler_cb; 2090 u16 masked_head, processed = 0; 2091 u8 phase; 2092 2093 masked_head = aenq->head & (aenq->q_depth - 1); 2094 phase = aenq->phase; 2095 aenq_e = &aenq->entries[masked_head]; /* Get first entry */ 2096 aenq_common = &aenq_e->aenq_common_desc; 2097 2098 /* Go over all the events */ 2099 while ((READ_ONCE8(aenq_common->flags) & 2100 ENA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK) == phase) { 2101 /* Make sure the phase bit (ownership) is as expected before 2102 * reading the rest of the descriptor. 2103 */ 2104 dma_rmb(); 2105 2106 timestamp = (u64)aenq_common->timestamp_low | 2107 ((u64)aenq_common->timestamp_high << 32); 2108 ENA_TOUCH(timestamp); /* In case debug is disabled */ 2109 ena_trc_dbg(ena_dev, "AENQ! Group[%x] Syndrome[%x] timestamp: [%" ENA_PRIu64 "s]\n", 2110 aenq_common->group, 2111 aenq_common->syndrome, 2112 timestamp); 2113 2114 /* Handle specific event*/ 2115 handler_cb = ena_com_get_specific_aenq_cb(ena_dev, 2116 aenq_common->group); 2117 handler_cb(data, aenq_e); /* call the actual event handler*/ 2118 2119 /* Get next event entry */ 2120 masked_head++; 2121 processed++; 2122 2123 if (unlikely(masked_head == aenq->q_depth)) { 2124 masked_head = 0; 2125 phase = !phase; 2126 } 2127 aenq_e = &aenq->entries[masked_head]; 2128 aenq_common = &aenq_e->aenq_common_desc; 2129 } 2130 2131 aenq->head += processed; 2132 aenq->phase = phase; 2133 2134 /* Don't update aenq doorbell if there weren't any processed events */ 2135 if (!processed) 2136 return; 2137 2138 /* write the aenq doorbell after all AENQ descriptors were read */ 2139 mb(); 2140 ENA_REG_WRITE32_RELAXED(ena_dev->bus, (u32)aenq->head, 2141 ena_dev->reg_bar + ENA_REGS_AENQ_HEAD_DB_OFF); 2142 mmiowb(); 2143 } 2144 2145 int ena_com_dev_reset(struct ena_com_dev *ena_dev, 2146 enum ena_regs_reset_reason_types reset_reason) 2147 { 2148 u32 stat, timeout, cap, reset_val; 2149 int rc; 2150 2151 stat = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF); 2152 cap = ena_com_reg_bar_read32(ena_dev, ENA_REGS_CAPS_OFF); 2153 2154 if (unlikely((stat == ENA_MMIO_READ_TIMEOUT) || 2155 (cap == ENA_MMIO_READ_TIMEOUT))) { 2156 ena_trc_err(ena_dev, "Reg read32 timeout occurred\n"); 2157 return ENA_COM_TIMER_EXPIRED; 2158 } 2159 2160 if ((stat & ENA_REGS_DEV_STS_READY_MASK) == 0) { 2161 ena_trc_err(ena_dev, "Device isn't ready, can't reset device\n"); 2162 return ENA_COM_INVAL; 2163 } 2164 2165 timeout = (cap & ENA_REGS_CAPS_RESET_TIMEOUT_MASK) >> 2166 ENA_REGS_CAPS_RESET_TIMEOUT_SHIFT; 2167 if (timeout == 0) { 2168 ena_trc_err(ena_dev, "Invalid timeout value\n"); 2169 return ENA_COM_INVAL; 2170 } 2171 2172 /* start reset */ 2173 reset_val = ENA_REGS_DEV_CTL_DEV_RESET_MASK; 2174 reset_val |= (reset_reason << ENA_REGS_DEV_CTL_RESET_REASON_SHIFT) & 2175 ENA_REGS_DEV_CTL_RESET_REASON_MASK; 2176 ENA_REG_WRITE32(ena_dev->bus, reset_val, ena_dev->reg_bar + ENA_REGS_DEV_CTL_OFF); 2177 2178 /* Write again the MMIO read request address */ 2179 ena_com_mmio_reg_read_request_write_dev_addr(ena_dev); 2180 2181 rc = wait_for_reset_state(ena_dev, timeout, 2182 ENA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK); 2183 if (rc != 0) { 2184 ena_trc_err(ena_dev, "Reset indication didn't turn on\n"); 2185 return rc; 2186 } 2187 2188 /* reset done */ 2189 ENA_REG_WRITE32(ena_dev->bus, 0, ena_dev->reg_bar + ENA_REGS_DEV_CTL_OFF); 2190 rc = wait_for_reset_state(ena_dev, timeout, 0); 2191 if (rc != 0) { 2192 ena_trc_err(ena_dev, "Reset indication didn't turn off\n"); 2193 return rc; 2194 } 2195 2196 timeout = (cap & ENA_REGS_CAPS_ADMIN_CMD_TO_MASK) >> 2197 ENA_REGS_CAPS_ADMIN_CMD_TO_SHIFT; 2198 if (timeout) 2199 /* the resolution of timeout reg is 100ms */ 2200 ena_dev->admin_queue.completion_timeout = timeout * 100000; 2201 else 2202 ena_dev->admin_queue.completion_timeout = ADMIN_CMD_TIMEOUT_US; 2203 2204 return 0; 2205 } 2206 2207 static int ena_get_dev_stats(struct ena_com_dev *ena_dev, 2208 struct ena_com_stats_ctx *ctx, 2209 enum ena_admin_get_stats_type type) 2210 { 2211 struct ena_admin_aq_get_stats_cmd *get_cmd = &ctx->get_cmd; 2212 struct ena_admin_acq_get_stats_resp *get_resp = &ctx->get_resp; 2213 struct ena_com_admin_queue *admin_queue; 2214 int ret; 2215 2216 admin_queue = &ena_dev->admin_queue; 2217 2218 get_cmd->aq_common_descriptor.opcode = ENA_ADMIN_GET_STATS; 2219 get_cmd->aq_common_descriptor.flags = 0; 2220 get_cmd->type = type; 2221 2222 ret = ena_com_execute_admin_command(admin_queue, 2223 (struct ena_admin_aq_entry *)get_cmd, 2224 sizeof(*get_cmd), 2225 (struct ena_admin_acq_entry *)get_resp, 2226 sizeof(*get_resp)); 2227 2228 if (unlikely(ret)) 2229 ena_trc_err(ena_dev, "Failed to get stats. error: %d\n", ret); 2230 2231 return ret; 2232 } 2233 2234 int ena_com_get_eni_stats(struct ena_com_dev *ena_dev, 2235 struct ena_admin_eni_stats *stats) 2236 { 2237 struct ena_com_stats_ctx ctx; 2238 int ret; 2239 2240 memset(&ctx, 0x0, sizeof(ctx)); 2241 ret = ena_get_dev_stats(ena_dev, &ctx, ENA_ADMIN_GET_STATS_TYPE_ENI); 2242 if (likely(ret == 0)) 2243 memcpy(stats, &ctx.get_resp.u.eni_stats, 2244 sizeof(ctx.get_resp.u.eni_stats)); 2245 2246 return ret; 2247 } 2248 2249 int ena_com_get_dev_basic_stats(struct ena_com_dev *ena_dev, 2250 struct ena_admin_basic_stats *stats) 2251 { 2252 struct ena_com_stats_ctx ctx; 2253 int ret; 2254 2255 memset(&ctx, 0x0, sizeof(ctx)); 2256 ret = ena_get_dev_stats(ena_dev, &ctx, ENA_ADMIN_GET_STATS_TYPE_BASIC); 2257 if (likely(ret == 0)) 2258 memcpy(stats, &ctx.get_resp.u.basic_stats, 2259 sizeof(ctx.get_resp.u.basic_stats)); 2260 2261 return ret; 2262 } 2263 2264 int ena_com_set_dev_mtu(struct ena_com_dev *ena_dev, int mtu) 2265 { 2266 struct ena_com_admin_queue *admin_queue; 2267 struct ena_admin_set_feat_cmd cmd; 2268 struct ena_admin_set_feat_resp resp; 2269 int ret; 2270 2271 if (!ena_com_check_supported_feature_id(ena_dev, ENA_ADMIN_MTU)) { 2272 ena_trc_dbg(ena_dev, "Feature %d isn't supported\n", ENA_ADMIN_MTU); 2273 return ENA_COM_UNSUPPORTED; 2274 } 2275 2276 memset(&cmd, 0x0, sizeof(cmd)); 2277 admin_queue = &ena_dev->admin_queue; 2278 2279 cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE; 2280 cmd.aq_common_descriptor.flags = 0; 2281 cmd.feat_common.feature_id = ENA_ADMIN_MTU; 2282 cmd.u.mtu.mtu = (u32)mtu; 2283 2284 ret = ena_com_execute_admin_command(admin_queue, 2285 (struct ena_admin_aq_entry *)&cmd, 2286 sizeof(cmd), 2287 (struct ena_admin_acq_entry *)&resp, 2288 sizeof(resp)); 2289 2290 if (unlikely(ret)) 2291 ena_trc_err(ena_dev, "Failed to set mtu %d. error: %d\n", mtu, ret); 2292 2293 return ret; 2294 } 2295 2296 int ena_com_get_offload_settings(struct ena_com_dev *ena_dev, 2297 struct ena_admin_feature_offload_desc *offload) 2298 { 2299 int ret; 2300 struct ena_admin_get_feat_resp resp; 2301 2302 ret = ena_com_get_feature(ena_dev, &resp, 2303 ENA_ADMIN_STATELESS_OFFLOAD_CONFIG, 0); 2304 if (unlikely(ret)) { 2305 ena_trc_err(ena_dev, "Failed to get offload capabilities %d\n", ret); 2306 return ret; 2307 } 2308 2309 memcpy(offload, &resp.u.offload, sizeof(resp.u.offload)); 2310 2311 return 0; 2312 } 2313 2314 int ena_com_set_hash_function(struct ena_com_dev *ena_dev) 2315 { 2316 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; 2317 struct ena_rss *rss = &ena_dev->rss; 2318 struct ena_admin_set_feat_cmd cmd; 2319 struct ena_admin_set_feat_resp resp; 2320 struct ena_admin_get_feat_resp get_resp; 2321 int ret; 2322 2323 if (!ena_com_check_supported_feature_id(ena_dev, 2324 ENA_ADMIN_RSS_HASH_FUNCTION)) { 2325 ena_trc_dbg(ena_dev, "Feature %d isn't supported\n", 2326 ENA_ADMIN_RSS_HASH_FUNCTION); 2327 return ENA_COM_UNSUPPORTED; 2328 } 2329 2330 /* Validate hash function is supported */ 2331 ret = ena_com_get_feature(ena_dev, &get_resp, 2332 ENA_ADMIN_RSS_HASH_FUNCTION, 0); 2333 if (unlikely(ret)) 2334 return ret; 2335 2336 if (!(get_resp.u.flow_hash_func.supported_func & BIT(rss->hash_func))) { 2337 ena_trc_err(ena_dev, "Func hash %d isn't supported by device, abort\n", 2338 rss->hash_func); 2339 return ENA_COM_UNSUPPORTED; 2340 } 2341 2342 memset(&cmd, 0x0, sizeof(cmd)); 2343 2344 cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE; 2345 cmd.aq_common_descriptor.flags = 2346 ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK; 2347 cmd.feat_common.feature_id = ENA_ADMIN_RSS_HASH_FUNCTION; 2348 cmd.u.flow_hash_func.init_val = rss->hash_init_val; 2349 cmd.u.flow_hash_func.selected_func = 1 << rss->hash_func; 2350 2351 ret = ena_com_mem_addr_set(ena_dev, 2352 &cmd.control_buffer.address, 2353 rss->hash_key_dma_addr); 2354 if (unlikely(ret)) { 2355 ena_trc_err(ena_dev, "Memory address set failed\n"); 2356 return ret; 2357 } 2358 2359 cmd.control_buffer.length = sizeof(*rss->hash_key); 2360 2361 ret = ena_com_execute_admin_command(admin_queue, 2362 (struct ena_admin_aq_entry *)&cmd, 2363 sizeof(cmd), 2364 (struct ena_admin_acq_entry *)&resp, 2365 sizeof(resp)); 2366 if (unlikely(ret)) { 2367 ena_trc_err(ena_dev, "Failed to set hash function %d. error: %d\n", 2368 rss->hash_func, ret); 2369 return ENA_COM_INVAL; 2370 } 2371 2372 return 0; 2373 } 2374 2375 int ena_com_fill_hash_function(struct ena_com_dev *ena_dev, 2376 enum ena_admin_hash_functions func, 2377 const u8 *key, u16 key_len, u32 init_val) 2378 { 2379 struct ena_admin_feature_rss_flow_hash_control *hash_key; 2380 struct ena_admin_get_feat_resp get_resp; 2381 enum ena_admin_hash_functions old_func; 2382 struct ena_rss *rss = &ena_dev->rss; 2383 int rc; 2384 2385 hash_key = rss->hash_key; 2386 2387 /* Make sure size is a mult of DWs */ 2388 if (unlikely(key_len & 0x3)) 2389 return ENA_COM_INVAL; 2390 2391 rc = ena_com_get_feature_ex(ena_dev, &get_resp, 2392 ENA_ADMIN_RSS_HASH_FUNCTION, 2393 rss->hash_key_dma_addr, 2394 sizeof(*rss->hash_key), 0); 2395 if (unlikely(rc)) 2396 return rc; 2397 2398 if (!(BIT(func) & get_resp.u.flow_hash_func.supported_func)) { 2399 ena_trc_err(ena_dev, "Flow hash function %d isn't supported\n", func); 2400 return ENA_COM_UNSUPPORTED; 2401 } 2402 2403 switch (func) { 2404 case ENA_ADMIN_TOEPLITZ: 2405 if (key) { 2406 if (key_len != sizeof(hash_key->key)) { 2407 ena_trc_err(ena_dev, "key len (%hu) doesn't equal the supported size (%zu)\n", 2408 key_len, sizeof(hash_key->key)); 2409 return ENA_COM_INVAL; 2410 } 2411 memcpy(hash_key->key, key, key_len); 2412 rss->hash_init_val = init_val; 2413 hash_key->key_parts = key_len / sizeof(hash_key->key[0]); 2414 } 2415 break; 2416 case ENA_ADMIN_CRC32: 2417 rss->hash_init_val = init_val; 2418 break; 2419 default: 2420 ena_trc_err(ena_dev, "Invalid hash function (%d)\n", func); 2421 return ENA_COM_INVAL; 2422 } 2423 2424 old_func = rss->hash_func; 2425 rss->hash_func = func; 2426 rc = ena_com_set_hash_function(ena_dev); 2427 2428 /* Restore the old function */ 2429 if (unlikely(rc)) 2430 rss->hash_func = old_func; 2431 2432 return rc; 2433 } 2434 2435 int ena_com_get_hash_function(struct ena_com_dev *ena_dev, 2436 enum ena_admin_hash_functions *func) 2437 { 2438 struct ena_rss *rss = &ena_dev->rss; 2439 struct ena_admin_get_feat_resp get_resp; 2440 int rc; 2441 2442 if (unlikely(!func)) 2443 return ENA_COM_INVAL; 2444 2445 rc = ena_com_get_feature_ex(ena_dev, &get_resp, 2446 ENA_ADMIN_RSS_HASH_FUNCTION, 2447 rss->hash_key_dma_addr, 2448 sizeof(*rss->hash_key), 0); 2449 if (unlikely(rc)) 2450 return rc; 2451 2452 /* ENA_FFS() returns 1 in case the lsb is set */ 2453 rss->hash_func = ENA_FFS(get_resp.u.flow_hash_func.selected_func); 2454 if (rss->hash_func) 2455 rss->hash_func--; 2456 2457 *func = rss->hash_func; 2458 2459 return 0; 2460 } 2461 2462 int ena_com_get_hash_key(struct ena_com_dev *ena_dev, u8 *key) 2463 { 2464 struct ena_admin_feature_rss_flow_hash_control *hash_key = 2465 ena_dev->rss.hash_key; 2466 2467 if (key) 2468 memcpy(key, hash_key->key, 2469 (size_t)(hash_key->key_parts) * sizeof(hash_key->key[0])); 2470 2471 return 0; 2472 } 2473 2474 int ena_com_get_hash_ctrl(struct ena_com_dev *ena_dev, 2475 enum ena_admin_flow_hash_proto proto, 2476 u16 *fields) 2477 { 2478 struct ena_rss *rss = &ena_dev->rss; 2479 struct ena_admin_get_feat_resp get_resp; 2480 int rc; 2481 2482 rc = ena_com_get_feature_ex(ena_dev, &get_resp, 2483 ENA_ADMIN_RSS_HASH_INPUT, 2484 rss->hash_ctrl_dma_addr, 2485 sizeof(*rss->hash_ctrl), 0); 2486 if (unlikely(rc)) 2487 return rc; 2488 2489 if (fields) 2490 *fields = rss->hash_ctrl->selected_fields[proto].fields; 2491 2492 return 0; 2493 } 2494 2495 int ena_com_set_hash_ctrl(struct ena_com_dev *ena_dev) 2496 { 2497 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; 2498 struct ena_rss *rss = &ena_dev->rss; 2499 struct ena_admin_feature_rss_hash_control *hash_ctrl = rss->hash_ctrl; 2500 struct ena_admin_set_feat_cmd cmd; 2501 struct ena_admin_set_feat_resp resp; 2502 int ret; 2503 2504 if (!ena_com_check_supported_feature_id(ena_dev, 2505 ENA_ADMIN_RSS_HASH_INPUT)) { 2506 ena_trc_dbg(ena_dev, "Feature %d isn't supported\n", 2507 ENA_ADMIN_RSS_HASH_INPUT); 2508 return ENA_COM_UNSUPPORTED; 2509 } 2510 2511 memset(&cmd, 0x0, sizeof(cmd)); 2512 2513 cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE; 2514 cmd.aq_common_descriptor.flags = 2515 ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK; 2516 cmd.feat_common.feature_id = ENA_ADMIN_RSS_HASH_INPUT; 2517 cmd.u.flow_hash_input.enabled_input_sort = 2518 ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_MASK | 2519 ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_MASK; 2520 2521 ret = ena_com_mem_addr_set(ena_dev, 2522 &cmd.control_buffer.address, 2523 rss->hash_ctrl_dma_addr); 2524 if (unlikely(ret)) { 2525 ena_trc_err(ena_dev, "Memory address set failed\n"); 2526 return ret; 2527 } 2528 cmd.control_buffer.length = sizeof(*hash_ctrl); 2529 2530 ret = ena_com_execute_admin_command(admin_queue, 2531 (struct ena_admin_aq_entry *)&cmd, 2532 sizeof(cmd), 2533 (struct ena_admin_acq_entry *)&resp, 2534 sizeof(resp)); 2535 if (unlikely(ret)) 2536 ena_trc_err(ena_dev, "Failed to set hash input. error: %d\n", ret); 2537 2538 return ret; 2539 } 2540 2541 int ena_com_set_default_hash_ctrl(struct ena_com_dev *ena_dev) 2542 { 2543 struct ena_rss *rss = &ena_dev->rss; 2544 struct ena_admin_feature_rss_hash_control *hash_ctrl = 2545 rss->hash_ctrl; 2546 u16 available_fields = 0; 2547 int rc, i; 2548 2549 /* Get the supported hash input */ 2550 rc = ena_com_get_hash_ctrl(ena_dev, 0, NULL); 2551 if (unlikely(rc)) 2552 return rc; 2553 2554 hash_ctrl->selected_fields[ENA_ADMIN_RSS_TCP4].fields = 2555 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA | 2556 ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP; 2557 2558 hash_ctrl->selected_fields[ENA_ADMIN_RSS_UDP4].fields = 2559 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA | 2560 ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP; 2561 2562 hash_ctrl->selected_fields[ENA_ADMIN_RSS_TCP6].fields = 2563 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA | 2564 ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP; 2565 2566 hash_ctrl->selected_fields[ENA_ADMIN_RSS_UDP6].fields = 2567 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA | 2568 ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP; 2569 2570 hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP4].fields = 2571 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA; 2572 2573 hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP6].fields = 2574 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA; 2575 2576 hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP4_FRAG].fields = 2577 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA; 2578 2579 hash_ctrl->selected_fields[ENA_ADMIN_RSS_NOT_IP].fields = 2580 ENA_ADMIN_RSS_L2_DA | ENA_ADMIN_RSS_L2_SA; 2581 2582 for (i = 0; i < ENA_ADMIN_RSS_PROTO_NUM; i++) { 2583 available_fields = hash_ctrl->selected_fields[i].fields & 2584 hash_ctrl->supported_fields[i].fields; 2585 if (available_fields != hash_ctrl->selected_fields[i].fields) { 2586 ena_trc_err(ena_dev, "Hash control doesn't support all the desire configuration. proto %x supported %x selected %x\n", 2587 i, hash_ctrl->supported_fields[i].fields, 2588 hash_ctrl->selected_fields[i].fields); 2589 return ENA_COM_UNSUPPORTED; 2590 } 2591 } 2592 2593 rc = ena_com_set_hash_ctrl(ena_dev); 2594 2595 /* In case of failure, restore the old hash ctrl */ 2596 if (unlikely(rc)) 2597 ena_com_get_hash_ctrl(ena_dev, 0, NULL); 2598 2599 return rc; 2600 } 2601 2602 int ena_com_fill_hash_ctrl(struct ena_com_dev *ena_dev, 2603 enum ena_admin_flow_hash_proto proto, 2604 u16 hash_fields) 2605 { 2606 struct ena_rss *rss = &ena_dev->rss; 2607 struct ena_admin_feature_rss_hash_control *hash_ctrl = rss->hash_ctrl; 2608 u16 supported_fields; 2609 int rc; 2610 2611 if (proto >= ENA_ADMIN_RSS_PROTO_NUM) { 2612 ena_trc_err(ena_dev, "Invalid proto num (%u)\n", proto); 2613 return ENA_COM_INVAL; 2614 } 2615 2616 /* Get the ctrl table */ 2617 rc = ena_com_get_hash_ctrl(ena_dev, proto, NULL); 2618 if (unlikely(rc)) 2619 return rc; 2620 2621 /* Make sure all the fields are supported */ 2622 supported_fields = hash_ctrl->supported_fields[proto].fields; 2623 if ((hash_fields & supported_fields) != hash_fields) { 2624 ena_trc_err(ena_dev, "Proto %d doesn't support the required fields %x. supports only: %x\n", 2625 proto, hash_fields, supported_fields); 2626 } 2627 2628 hash_ctrl->selected_fields[proto].fields = hash_fields; 2629 2630 rc = ena_com_set_hash_ctrl(ena_dev); 2631 2632 /* In case of failure, restore the old hash ctrl */ 2633 if (unlikely(rc)) 2634 ena_com_get_hash_ctrl(ena_dev, 0, NULL); 2635 2636 return 0; 2637 } 2638 2639 int ena_com_indirect_table_fill_entry(struct ena_com_dev *ena_dev, 2640 u16 entry_idx, u16 entry_value) 2641 { 2642 struct ena_rss *rss = &ena_dev->rss; 2643 2644 if (unlikely(entry_idx >= (1 << rss->tbl_log_size))) 2645 return ENA_COM_INVAL; 2646 2647 if (unlikely((entry_value > ENA_TOTAL_NUM_QUEUES))) 2648 return ENA_COM_INVAL; 2649 2650 rss->host_rss_ind_tbl[entry_idx] = entry_value; 2651 2652 return 0; 2653 } 2654 2655 int ena_com_indirect_table_set(struct ena_com_dev *ena_dev) 2656 { 2657 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; 2658 struct ena_rss *rss = &ena_dev->rss; 2659 struct ena_admin_set_feat_cmd cmd; 2660 struct ena_admin_set_feat_resp resp; 2661 int ret; 2662 2663 if (!ena_com_check_supported_feature_id(ena_dev, 2664 ENA_ADMIN_RSS_INDIRECTION_TABLE_CONFIG)) { 2665 ena_trc_dbg(ena_dev, "Feature %d isn't supported\n", 2666 ENA_ADMIN_RSS_INDIRECTION_TABLE_CONFIG); 2667 return ENA_COM_UNSUPPORTED; 2668 } 2669 2670 ret = ena_com_ind_tbl_convert_to_device(ena_dev); 2671 if (ret) { 2672 ena_trc_err(ena_dev, "Failed to convert host indirection table to device table\n"); 2673 return ret; 2674 } 2675 2676 memset(&cmd, 0x0, sizeof(cmd)); 2677 2678 cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE; 2679 cmd.aq_common_descriptor.flags = 2680 ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK; 2681 cmd.feat_common.feature_id = ENA_ADMIN_RSS_INDIRECTION_TABLE_CONFIG; 2682 cmd.u.ind_table.size = rss->tbl_log_size; 2683 cmd.u.ind_table.inline_index = 0xFFFFFFFF; 2684 2685 ret = ena_com_mem_addr_set(ena_dev, 2686 &cmd.control_buffer.address, 2687 rss->rss_ind_tbl_dma_addr); 2688 if (unlikely(ret)) { 2689 ena_trc_err(ena_dev, "Memory address set failed\n"); 2690 return ret; 2691 } 2692 2693 cmd.control_buffer.length = (u32)(1ULL << rss->tbl_log_size) * 2694 sizeof(struct ena_admin_rss_ind_table_entry); 2695 2696 ret = ena_com_execute_admin_command(admin_queue, 2697 (struct ena_admin_aq_entry *)&cmd, 2698 sizeof(cmd), 2699 (struct ena_admin_acq_entry *)&resp, 2700 sizeof(resp)); 2701 2702 if (unlikely(ret)) 2703 ena_trc_err(ena_dev, "Failed to set indirect table. error: %d\n", ret); 2704 2705 return ret; 2706 } 2707 2708 int ena_com_indirect_table_get(struct ena_com_dev *ena_dev, u32 *ind_tbl) 2709 { 2710 struct ena_rss *rss = &ena_dev->rss; 2711 struct ena_admin_get_feat_resp get_resp; 2712 u32 tbl_size; 2713 int i, rc; 2714 2715 tbl_size = (u32)(1ULL << rss->tbl_log_size) * 2716 sizeof(struct ena_admin_rss_ind_table_entry); 2717 2718 rc = ena_com_get_feature_ex(ena_dev, &get_resp, 2719 ENA_ADMIN_RSS_INDIRECTION_TABLE_CONFIG, 2720 rss->rss_ind_tbl_dma_addr, 2721 tbl_size, 0); 2722 if (unlikely(rc)) 2723 return rc; 2724 2725 if (!ind_tbl) 2726 return 0; 2727 2728 for (i = 0; i < (1 << rss->tbl_log_size); i++) 2729 ind_tbl[i] = rss->host_rss_ind_tbl[i]; 2730 2731 return 0; 2732 } 2733 2734 int ena_com_rss_init(struct ena_com_dev *ena_dev, u16 indr_tbl_log_size) 2735 { 2736 int rc; 2737 2738 memset(&ena_dev->rss, 0x0, sizeof(ena_dev->rss)); 2739 2740 rc = ena_com_indirect_table_allocate(ena_dev, indr_tbl_log_size); 2741 if (unlikely(rc)) 2742 goto err_indr_tbl; 2743 2744 /* The following function might return unsupported in case the 2745 * device doesn't support setting the key / hash function. We can safely 2746 * ignore this error and have indirection table support only. 2747 */ 2748 rc = ena_com_hash_key_allocate(ena_dev); 2749 if (likely(!rc)) 2750 ena_com_hash_key_fill_default_key(ena_dev); 2751 else if (rc != ENA_COM_UNSUPPORTED) 2752 goto err_hash_key; 2753 2754 rc = ena_com_hash_ctrl_init(ena_dev); 2755 if (unlikely(rc)) 2756 goto err_hash_ctrl; 2757 2758 return 0; 2759 2760 err_hash_ctrl: 2761 ena_com_hash_key_destroy(ena_dev); 2762 err_hash_key: 2763 ena_com_indirect_table_destroy(ena_dev); 2764 err_indr_tbl: 2765 2766 return rc; 2767 } 2768 2769 void ena_com_rss_destroy(struct ena_com_dev *ena_dev) 2770 { 2771 ena_com_indirect_table_destroy(ena_dev); 2772 ena_com_hash_key_destroy(ena_dev); 2773 ena_com_hash_ctrl_destroy(ena_dev); 2774 2775 memset(&ena_dev->rss, 0x0, sizeof(ena_dev->rss)); 2776 } 2777 2778 int ena_com_allocate_host_info(struct ena_com_dev *ena_dev) 2779 { 2780 struct ena_host_attribute *host_attr = &ena_dev->host_attr; 2781 2782 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, 2783 SZ_4K, 2784 host_attr->host_info, 2785 host_attr->host_info_dma_addr, 2786 host_attr->host_info_dma_handle); 2787 if (unlikely(!host_attr->host_info)) 2788 return ENA_COM_NO_MEM; 2789 2790 host_attr->host_info->ena_spec_version = ((ENA_COMMON_SPEC_VERSION_MAJOR << 2791 ENA_REGS_VERSION_MAJOR_VERSION_SHIFT) | 2792 (ENA_COMMON_SPEC_VERSION_MINOR)); 2793 2794 return 0; 2795 } 2796 2797 int ena_com_allocate_debug_area(struct ena_com_dev *ena_dev, 2798 u32 debug_area_size) 2799 { 2800 struct ena_host_attribute *host_attr = &ena_dev->host_attr; 2801 2802 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, 2803 debug_area_size, 2804 host_attr->debug_area_virt_addr, 2805 host_attr->debug_area_dma_addr, 2806 host_attr->debug_area_dma_handle); 2807 if (unlikely(!host_attr->debug_area_virt_addr)) { 2808 host_attr->debug_area_size = 0; 2809 return ENA_COM_NO_MEM; 2810 } 2811 2812 host_attr->debug_area_size = debug_area_size; 2813 2814 return 0; 2815 } 2816 2817 void ena_com_delete_host_info(struct ena_com_dev *ena_dev) 2818 { 2819 struct ena_host_attribute *host_attr = &ena_dev->host_attr; 2820 2821 if (host_attr->host_info) { 2822 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, 2823 SZ_4K, 2824 host_attr->host_info, 2825 host_attr->host_info_dma_addr, 2826 host_attr->host_info_dma_handle); 2827 host_attr->host_info = NULL; 2828 } 2829 } 2830 2831 void ena_com_delete_debug_area(struct ena_com_dev *ena_dev) 2832 { 2833 struct ena_host_attribute *host_attr = &ena_dev->host_attr; 2834 2835 if (host_attr->debug_area_virt_addr) { 2836 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, 2837 host_attr->debug_area_size, 2838 host_attr->debug_area_virt_addr, 2839 host_attr->debug_area_dma_addr, 2840 host_attr->debug_area_dma_handle); 2841 host_attr->debug_area_virt_addr = NULL; 2842 } 2843 } 2844 2845 int ena_com_set_host_attributes(struct ena_com_dev *ena_dev) 2846 { 2847 struct ena_host_attribute *host_attr = &ena_dev->host_attr; 2848 struct ena_com_admin_queue *admin_queue; 2849 struct ena_admin_set_feat_cmd cmd; 2850 struct ena_admin_set_feat_resp resp; 2851 2852 int ret; 2853 2854 /* Host attribute config is called before ena_com_get_dev_attr_feat 2855 * so ena_com can't check if the feature is supported. 2856 */ 2857 2858 memset(&cmd, 0x0, sizeof(cmd)); 2859 admin_queue = &ena_dev->admin_queue; 2860 2861 cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE; 2862 cmd.feat_common.feature_id = ENA_ADMIN_HOST_ATTR_CONFIG; 2863 2864 ret = ena_com_mem_addr_set(ena_dev, 2865 &cmd.u.host_attr.debug_ba, 2866 host_attr->debug_area_dma_addr); 2867 if (unlikely(ret)) { 2868 ena_trc_err(ena_dev, "Memory address set failed\n"); 2869 return ret; 2870 } 2871 2872 ret = ena_com_mem_addr_set(ena_dev, 2873 &cmd.u.host_attr.os_info_ba, 2874 host_attr->host_info_dma_addr); 2875 if (unlikely(ret)) { 2876 ena_trc_err(ena_dev, "Memory address set failed\n"); 2877 return ret; 2878 } 2879 2880 cmd.u.host_attr.debug_area_size = host_attr->debug_area_size; 2881 2882 ret = ena_com_execute_admin_command(admin_queue, 2883 (struct ena_admin_aq_entry *)&cmd, 2884 sizeof(cmd), 2885 (struct ena_admin_acq_entry *)&resp, 2886 sizeof(resp)); 2887 2888 if (unlikely(ret)) 2889 ena_trc_err(ena_dev, "Failed to set host attributes: %d\n", ret); 2890 2891 return ret; 2892 } 2893 2894 /* Interrupt moderation */ 2895 bool ena_com_interrupt_moderation_supported(struct ena_com_dev *ena_dev) 2896 { 2897 return ena_com_check_supported_feature_id(ena_dev, 2898 ENA_ADMIN_INTERRUPT_MODERATION); 2899 } 2900 2901 static int ena_com_update_nonadaptive_moderation_interval(struct ena_com_dev *ena_dev, 2902 u32 coalesce_usecs, 2903 u32 intr_delay_resolution, 2904 u32 *intr_moder_interval) 2905 { 2906 if (!intr_delay_resolution) { 2907 ena_trc_err(ena_dev, "Illegal interrupt delay granularity value\n"); 2908 return ENA_COM_FAULT; 2909 } 2910 2911 *intr_moder_interval = coalesce_usecs / intr_delay_resolution; 2912 2913 return 0; 2914 } 2915 2916 int ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev, 2917 u32 tx_coalesce_usecs) 2918 { 2919 return ena_com_update_nonadaptive_moderation_interval(ena_dev, 2920 tx_coalesce_usecs, 2921 ena_dev->intr_delay_resolution, 2922 &ena_dev->intr_moder_tx_interval); 2923 } 2924 2925 int ena_com_update_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev, 2926 u32 rx_coalesce_usecs) 2927 { 2928 return ena_com_update_nonadaptive_moderation_interval(ena_dev, 2929 rx_coalesce_usecs, 2930 ena_dev->intr_delay_resolution, 2931 &ena_dev->intr_moder_rx_interval); 2932 } 2933 2934 int ena_com_init_interrupt_moderation(struct ena_com_dev *ena_dev) 2935 { 2936 struct ena_admin_get_feat_resp get_resp; 2937 u16 delay_resolution; 2938 int rc; 2939 2940 rc = ena_com_get_feature(ena_dev, &get_resp, 2941 ENA_ADMIN_INTERRUPT_MODERATION, 0); 2942 2943 if (rc) { 2944 if (rc == ENA_COM_UNSUPPORTED) { 2945 ena_trc_dbg(ena_dev, "Feature %d isn't supported\n", 2946 ENA_ADMIN_INTERRUPT_MODERATION); 2947 rc = 0; 2948 } else { 2949 ena_trc_err(ena_dev, 2950 "Failed to get interrupt moderation admin cmd. rc: %d\n", rc); 2951 } 2952 2953 /* no moderation supported, disable adaptive support */ 2954 ena_com_disable_adaptive_moderation(ena_dev); 2955 return rc; 2956 } 2957 2958 /* if moderation is supported by device we set adaptive moderation */ 2959 delay_resolution = get_resp.u.intr_moderation.intr_delay_resolution; 2960 ena_com_update_intr_delay_resolution(ena_dev, delay_resolution); 2961 2962 /* Disable adaptive moderation by default - can be enabled later */ 2963 ena_com_disable_adaptive_moderation(ena_dev); 2964 2965 return 0; 2966 } 2967 2968 unsigned int ena_com_get_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev) 2969 { 2970 return ena_dev->intr_moder_tx_interval; 2971 } 2972 2973 unsigned int ena_com_get_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev) 2974 { 2975 return ena_dev->intr_moder_rx_interval; 2976 } 2977 2978 int ena_com_config_dev_mode(struct ena_com_dev *ena_dev, 2979 struct ena_admin_feature_llq_desc *llq_features, 2980 struct ena_llq_configurations *llq_default_cfg) 2981 { 2982 struct ena_com_llq_info *llq_info = &ena_dev->llq_info; 2983 int rc; 2984 2985 if (!llq_features->max_llq_num) { 2986 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 2987 return 0; 2988 } 2989 2990 rc = ena_com_config_llq_info(ena_dev, llq_features, llq_default_cfg); 2991 if (rc) 2992 return rc; 2993 2994 ena_dev->tx_max_header_size = llq_info->desc_list_entry_size - 2995 (llq_info->descs_num_before_header * sizeof(struct ena_eth_io_tx_desc)); 2996 2997 if (unlikely(ena_dev->tx_max_header_size == 0)) { 2998 ena_trc_err(ena_dev, "The size of the LLQ entry is smaller than needed\n"); 2999 return -EINVAL; 3000 } 3001 3002 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_DEV; 3003 3004 return 0; 3005 } 3006