119 static int ena_com_admin_init_sq(struct ena_com_admin_queue *admin_queue)  in ena_com_admin_init_sq()  argument
121 	struct ena_com_dev *ena_dev = admin_queue->ena_dev;  in ena_com_admin_init_sq()
122 	struct ena_com_admin_sq *sq = &admin_queue->sq;  in ena_com_admin_init_sq()
123 	u16 size = ADMIN_SQ_SIZE(admin_queue->q_depth);  in ena_com_admin_init_sq()
125 	ENA_MEM_ALLOC_COHERENT(admin_queue->q_dmadev, size, sq->entries, sq->dma_addr,  in ena_com_admin_init_sq()
142 static int ena_com_admin_init_cq(struct ena_com_admin_queue *admin_queue)  in ena_com_admin_init_cq()  argument
144 	struct ena_com_dev *ena_dev = admin_queue->ena_dev;  in ena_com_admin_init_cq()
145 	struct ena_com_admin_cq *cq = &admin_queue->cq;  in ena_com_admin_init_cq()
146 	u16 size = ADMIN_CQ_SIZE(admin_queue->q_depth);  in ena_com_admin_init_cq()
148 	ENA_MEM_ALLOC_COHERENT(admin_queue->q_dmadev, size, cq->entries, cq->dma_addr,  in ena_com_admin_init_cq()
214 static struct ena_comp_ctx *get_comp_ctxt(struct ena_com_admin_queue *admin_queue,  in get_comp_ctxt()  argument
217 	if (unlikely(command_id >= admin_queue->q_depth)) {  in get_comp_ctxt()
218 		ena_trc_err(admin_queue->ena_dev,  in get_comp_ctxt()
220 			    command_id, admin_queue->q_depth);  in get_comp_ctxt()
224 	if (unlikely(!admin_queue->comp_ctx)) {  in get_comp_ctxt()
225 		ena_trc_err(admin_queue->ena_dev,  in get_comp_ctxt()
230 	if (unlikely(admin_queue->comp_ctx[command_id].occupied && capture)) {  in get_comp_ctxt()
231 		ena_trc_err(admin_queue->ena_dev,  in get_comp_ctxt()
237 		ATOMIC32_INC(&admin_queue->outstanding_cmds);  in get_comp_ctxt()
238 		admin_queue->comp_ctx[command_id].occupied = true;  in get_comp_ctxt()
241 	return &admin_queue->comp_ctx[command_id];  in get_comp_ctxt()
244 static struct ena_comp_ctx *__ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,  in __ena_com_submit_admin_cmd()  argument
255 	queue_size_mask = admin_queue->q_depth - 1;  in __ena_com_submit_admin_cmd()
257 	tail_masked = admin_queue->sq.tail & queue_size_mask;  in __ena_com_submit_admin_cmd()
260 	cnt = (u16)ATOMIC32_READ(&admin_queue->outstanding_cmds);  in __ena_com_submit_admin_cmd()
261 	if (cnt >= admin_queue->q_depth) {  in __ena_com_submit_admin_cmd()
262 		ena_trc_dbg(admin_queue->ena_dev, "Admin queue is full.\n");  in __ena_com_submit_admin_cmd()
263 		admin_queue->stats.out_of_space++;  in __ena_com_submit_admin_cmd()
267 	cmd_id = admin_queue->curr_cmd_id;  in __ena_com_submit_admin_cmd()
269 	cmd->aq_common_descriptor.flags |= admin_queue->sq.phase &  in __ena_com_submit_admin_cmd()
275 	comp_ctx = get_comp_ctxt(admin_queue, cmd_id, true);  in __ena_com_submit_admin_cmd()
286 	memcpy(&admin_queue->sq.entries[tail_masked], cmd, cmd_size_in_bytes);  in __ena_com_submit_admin_cmd()
288 	admin_queue->curr_cmd_id = (admin_queue->curr_cmd_id + 1) &  in __ena_com_submit_admin_cmd()
291 	admin_queue->sq.tail++;  in __ena_com_submit_admin_cmd()
292 	admin_queue->stats.submitted_cmd++;  in __ena_com_submit_admin_cmd()
294 	if (unlikely((admin_queue->sq.tail & queue_size_mask) == 0))  in __ena_com_submit_admin_cmd()
295 		admin_queue->sq.phase = !admin_queue->sq.phase;  in __ena_com_submit_admin_cmd()
297 	ENA_DB_SYNC(&admin_queue->sq.mem_handle);  in __ena_com_submit_admin_cmd()
298 	ENA_REG_WRITE32(admin_queue->bus, admin_queue->sq.tail,  in __ena_com_submit_admin_cmd()
299 			admin_queue->sq.db_addr);  in __ena_com_submit_admin_cmd()
304 static int ena_com_init_comp_ctxt(struct ena_com_admin_queue *admin_queue)  in ena_com_init_comp_ctxt()  argument
306 	struct ena_com_dev *ena_dev = admin_queue->ena_dev;  in ena_com_init_comp_ctxt()
307 	size_t size = admin_queue->q_depth * sizeof(struct ena_comp_ctx);  in ena_com_init_comp_ctxt()
311 	admin_queue->comp_ctx = ENA_MEM_ALLOC(admin_queue->q_dmadev, size);  in ena_com_init_comp_ctxt()
312 	if (unlikely(!admin_queue->comp_ctx)) {  in ena_com_init_comp_ctxt()
317 	for (i = 0; i < admin_queue->q_depth; i++) {  in ena_com_init_comp_ctxt()
318 		comp_ctx = get_comp_ctxt(admin_queue, i, false);  in ena_com_init_comp_ctxt()
326 static struct ena_comp_ctx *ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,  in ena_com_submit_admin_cmd()  argument
335 	ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
336 	if (unlikely(!admin_queue->running_state)) {  in ena_com_submit_admin_cmd()
337 		ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
340 	comp_ctx = __ena_com_submit_admin_cmd(admin_queue, cmd,  in ena_com_submit_admin_cmd()
345 		admin_queue->running_state = false;  in ena_com_submit_admin_cmd()
346 	ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
486 static void ena_com_handle_single_admin_completion(struct ena_com_admin_queue *admin_queue,  in ena_com_handle_single_admin_completion()  argument
495 	comp_ctx = get_comp_ctxt(admin_queue, cmd_id, false);  in ena_com_handle_single_admin_completion()
497 		ena_trc_err(admin_queue->ena_dev,  in ena_com_handle_single_admin_completion()
499 		admin_queue->running_state = false;  in ena_com_handle_single_admin_completion()
509 	if (!admin_queue->polling)  in ena_com_handle_single_admin_completion()
513 static void ena_com_handle_admin_completion(struct ena_com_admin_queue *admin_queue)  in ena_com_handle_admin_completion()  argument
520 	head_masked = admin_queue->cq.head & (admin_queue->q_depth - 1);  in ena_com_handle_admin_completion()
521 	phase = admin_queue->cq.phase;  in ena_com_handle_admin_completion()
523 	cqe = &admin_queue->cq.entries[head_masked];  in ena_com_handle_admin_completion()
532 		ena_com_handle_single_admin_completion(admin_queue, cqe);  in ena_com_handle_admin_completion()
536 		if (unlikely(head_masked == admin_queue->q_depth)) {  in ena_com_handle_admin_completion()
541 		cqe = &admin_queue->cq.entries[head_masked];  in ena_com_handle_admin_completion()
544 	admin_queue->cq.head += comp_num;  in ena_com_handle_admin_completion()
545 	admin_queue->cq.phase = phase;  in ena_com_handle_admin_completion()
546 	admin_queue->sq.head += comp_num;  in ena_com_handle_admin_completion()
547 	admin_queue->stats.completed_cmd += comp_num;  in ena_com_handle_admin_completion()
550 static int ena_com_comp_status_to_errno(struct ena_com_admin_queue *admin_queue,  in ena_com_comp_status_to_errno()  argument
554 		ena_trc_err(admin_queue->ena_dev,  in ena_com_comp_status_to_errno()
584 						     struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq_polling()  argument
591 	timeout = ENA_GET_SYSTEM_TIMEOUT(admin_queue->completion_timeout);  in ena_com_wait_and_process_admin_cq_polling()
594 		ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
595 		ena_com_handle_admin_completion(admin_queue);  in ena_com_wait_and_process_admin_cq_polling()
596 		ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
602 			ena_trc_err(admin_queue->ena_dev,  in ena_com_wait_and_process_admin_cq_polling()
605 			ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
606 			admin_queue->stats.no_completion++;  in ena_com_wait_and_process_admin_cq_polling()
607 			admin_queue->running_state = false;  in ena_com_wait_and_process_admin_cq_polling()
608 			ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
615 						 admin_queue->ena_dev->ena_min_poll_delay_us);  in ena_com_wait_and_process_admin_cq_polling()
619 		ena_trc_err(admin_queue->ena_dev, "Command was aborted\n");  in ena_com_wait_and_process_admin_cq_polling()
620 		ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
621 		admin_queue->stats.aborted_cmd++;  in ena_com_wait_and_process_admin_cq_polling()
622 		ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
628 		 admin_queue->ena_dev, "Invalid comp status %d\n",  in ena_com_wait_and_process_admin_cq_polling()
631 	ret = ena_com_comp_status_to_errno(admin_queue, comp_ctx->comp_status);  in ena_com_wait_and_process_admin_cq_polling()
633 	comp_ctxt_release(admin_queue, comp_ctx);  in ena_com_wait_and_process_admin_cq_polling()
645 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_llq()  local
652 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_llq()
666 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_llq()
807 							struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq_interrupts()  argument
813 			    admin_queue->completion_timeout);  in ena_com_wait_and_process_admin_cq_interrupts()
821 		ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_interrupts()
822 		ena_com_handle_admin_completion(admin_queue);  in ena_com_wait_and_process_admin_cq_interrupts()
823 		admin_queue->stats.no_completion++;  in ena_com_wait_and_process_admin_cq_interrupts()
824 		ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_interrupts()
827 			ena_trc_err(admin_queue->ena_dev,  in ena_com_wait_and_process_admin_cq_interrupts()
829 				    comp_ctx->cmd_opcode, admin_queue->auto_polling ? "ON" : "OFF");  in ena_com_wait_and_process_admin_cq_interrupts()
831 			if (admin_queue->auto_polling)  in ena_com_wait_and_process_admin_cq_interrupts()
832 				admin_queue->polling = true;  in ena_com_wait_and_process_admin_cq_interrupts()
834 			ena_trc_err(admin_queue->ena_dev,  in ena_com_wait_and_process_admin_cq_interrupts()
842 		if (!admin_queue->polling) {  in ena_com_wait_and_process_admin_cq_interrupts()
843 			admin_queue->running_state = false;  in ena_com_wait_and_process_admin_cq_interrupts()
849 	ret = ena_com_comp_status_to_errno(admin_queue, comp_ctx->comp_status);  in ena_com_wait_and_process_admin_cq_interrupts()
851 	comp_ctxt_release(admin_queue, comp_ctx);  in ena_com_wait_and_process_admin_cq_interrupts()
926 					     struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq()  argument
928 	if (admin_queue->polling)  in ena_com_wait_and_process_admin_cq()
930 								 admin_queue);  in ena_com_wait_and_process_admin_cq()
933 							    admin_queue);  in ena_com_wait_and_process_admin_cq()
939 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_destroy_io_sq()  local
959 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_destroy_io_sq()
1057 	struct ena_com_admin_queue *admin_queue;  in ena_com_get_feature_ex()  local
1067 	admin_queue = &ena_dev->admin_queue;  in ena_com_get_feature_ex()
1089 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_get_feature_ex()
1277 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_create_io_sq()  local
1319 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_create_io_sq()
1400 int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue,  in ena_com_execute_admin_command()  argument
1409 	comp_ctx = ena_com_submit_admin_cmd(admin_queue, cmd, cmd_size,  in ena_com_execute_admin_command()
1413 			ena_trc_dbg(admin_queue->ena_dev,  in ena_com_execute_admin_command()
1417 			ena_trc_err(admin_queue->ena_dev,  in ena_com_execute_admin_command()
1424 	ret = ena_com_wait_and_process_admin_cq(comp_ctx, admin_queue);  in ena_com_execute_admin_command()
1426 		if (admin_queue->running_state)  in ena_com_execute_admin_command()
1427 			ena_trc_err(admin_queue->ena_dev,  in ena_com_execute_admin_command()
1430 			ena_trc_dbg(admin_queue->ena_dev,  in ena_com_execute_admin_command()
1439 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_create_io_cq()  local
1464 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_create_io_cq()
1512 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_abort_admin_commands()  local
1516 	if (!admin_queue->comp_ctx)  in ena_com_abort_admin_commands()
1519 	for (i = 0; i < admin_queue->q_depth; i++) {  in ena_com_abort_admin_commands()
1520 		comp_ctx = get_comp_ctxt(admin_queue, i, false);  in ena_com_abort_admin_commands()
1532 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_wait_for_abort_completion()  local
1536 	ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1537 	while (ATOMIC32_READ(&admin_queue->outstanding_cmds) != 0) {  in ena_com_wait_for_abort_completion()
1538 		ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1540 		ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1542 	ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1548 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_destroy_io_cq()  local
1558 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_destroy_io_cq()
1572 	return ena_dev->admin_queue.running_state;  in ena_com_get_admin_running_state()
1577 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_admin_running_state()  local
1580 	ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_set_admin_running_state()
1581 	ena_dev->admin_queue.running_state = state;  in ena_com_set_admin_running_state()
1582 	ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_set_admin_running_state()
1599 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_aenq_config()  local
1619 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_aenq_config()
1626 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_aenq_config()
1712 				      struct ena_com_admin_queue *admin_queue)  in ena_com_free_ena_admin_queue_comp_ctx()  argument
1715 	if (!admin_queue->comp_ctx)  in ena_com_free_ena_admin_queue_comp_ctx()
1718 	ENA_WAIT_EVENTS_DESTROY(admin_queue);  in ena_com_free_ena_admin_queue_comp_ctx()
1720 		     admin_queue->comp_ctx,  in ena_com_free_ena_admin_queue_comp_ctx()
1721 		     (admin_queue->q_depth * sizeof(struct ena_comp_ctx)));  in ena_com_free_ena_admin_queue_comp_ctx()
1723 	admin_queue->comp_ctx = NULL;  in ena_com_free_ena_admin_queue_comp_ctx()
1728 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_admin_destroy()  local
1729 	struct ena_com_admin_cq *cq = &admin_queue->cq;  in ena_com_admin_destroy()
1730 	struct ena_com_admin_sq *sq = &admin_queue->sq;  in ena_com_admin_destroy()
1734 	ena_com_free_ena_admin_queue_comp_ctx(ena_dev, admin_queue);  in ena_com_admin_destroy()
1736 	size = ADMIN_SQ_SIZE(admin_queue->q_depth);  in ena_com_admin_destroy()
1742 	size = ADMIN_CQ_SIZE(admin_queue->q_depth);  in ena_com_admin_destroy()
1753 	ENA_SPINLOCK_DESTROY(admin_queue->q_lock);  in ena_com_admin_destroy()
1765 	ena_dev->admin_queue.polling = polling;  in ena_com_set_admin_polling_mode()
1770 	return ena_dev->admin_queue.polling;  in ena_com_get_admin_polling_mode()
1776 	ena_dev->admin_queue.auto_polling = polling;  in ena_com_set_admin_auto_polling_mode()
1844 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_admin_init()  local
1860 	admin_queue->q_depth = ENA_ADMIN_QUEUE_DEPTH;  in ena_com_admin_init()
1862 	admin_queue->bus = ena_dev->bus;  in ena_com_admin_init()
1863 	admin_queue->q_dmadev = ena_dev->dmadev;  in ena_com_admin_init()
1864 	admin_queue->polling = false;  in ena_com_admin_init()
1865 	admin_queue->curr_cmd_id = 0;  in ena_com_admin_init()
1867 	ATOMIC32_SET(&admin_queue->outstanding_cmds, 0);  in ena_com_admin_init()
1869 	ENA_SPINLOCK_INIT(admin_queue->q_lock);  in ena_com_admin_init()
1871 	ret = ena_com_init_comp_ctxt(admin_queue);  in ena_com_admin_init()
1875 	ret = ena_com_admin_init_sq(admin_queue);  in ena_com_admin_init()
1879 	ret = ena_com_admin_init_cq(admin_queue);  in ena_com_admin_init()
1883 	admin_queue->sq.db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +  in ena_com_admin_init()
1886 	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->sq.dma_addr);  in ena_com_admin_init()
1887 	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->sq.dma_addr);  in ena_com_admin_init()
1892 	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->cq.dma_addr);  in ena_com_admin_init()
1893 	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->cq.dma_addr);  in ena_com_admin_init()
1899 	aq_caps |= admin_queue->q_depth & ENA_REGS_AQ_CAPS_AQ_DEPTH_MASK;  in ena_com_admin_init()
1905 	acq_caps |= admin_queue->q_depth & ENA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK;  in ena_com_admin_init()
1916 	admin_queue->ena_dev = ena_dev;  in ena_com_admin_init()
1917 	admin_queue->running_state = true;  in ena_com_admin_init()
2098 	ena_com_handle_admin_completion(&ena_dev->admin_queue);  in ena_com_admin_q_comp_intr_handler()
2257 		ena_dev->admin_queue.completion_timeout = timeout * 100000;  in ena_com_dev_reset()
2259 		ena_dev->admin_queue.completion_timeout = ADMIN_CMD_TIMEOUT_US;  in ena_com_dev_reset()
2270 	struct ena_com_admin_queue *admin_queue;  in ena_get_dev_stats()  local
2273 	admin_queue = &ena_dev->admin_queue;  in ena_get_dev_stats()
2279 	ret =  ena_com_execute_admin_command(admin_queue,  in ena_get_dev_stats()
2368 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_dev_mtu()  local
2379 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_dev_mtu()
2386 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_dev_mtu()
2418 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_hash_function()  local
2463 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_hash_function()
2599 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_hash_ctrl()  local
2632 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_hash_ctrl()
2759 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_indirect_table_set()  local
2798 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_indirect_table_set()
2950 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_host_attributes()  local
2961 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_host_attributes()
2984 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_host_attributes()