203 static struct ena_comp_ctx *__ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,  in __ena_com_submit_admin_cmd()  argument
214 	queue_size_mask = admin_queue->q_depth - 1;  in __ena_com_submit_admin_cmd()
216 	tail_masked = admin_queue->sq.tail & queue_size_mask;  in __ena_com_submit_admin_cmd()
219 	cnt = (u16)ATOMIC32_READ(&admin_queue->outstanding_cmds);  in __ena_com_submit_admin_cmd()
220 	if (cnt >= admin_queue->q_depth) {  in __ena_com_submit_admin_cmd()
222 		admin_queue->stats.out_of_space++;  in __ena_com_submit_admin_cmd()
226 	cmd_id = admin_queue->curr_cmd_id;  in __ena_com_submit_admin_cmd()
228 	cmd->aq_common_descriptor.flags |= admin_queue->sq.phase &  in __ena_com_submit_admin_cmd()
234 	comp_ctx = get_comp_ctxt(admin_queue, cmd_id, true);  in __ena_com_submit_admin_cmd()
245 	memcpy(&admin_queue->sq.entries[tail_masked], cmd, cmd_size_in_bytes);  in __ena_com_submit_admin_cmd()
247 	admin_queue->curr_cmd_id = (admin_queue->curr_cmd_id + 1) &  in __ena_com_submit_admin_cmd()
250 	admin_queue->sq.tail++;  in __ena_com_submit_admin_cmd()
251 	admin_queue->stats.submitted_cmd++;  in __ena_com_submit_admin_cmd()
253 	if (unlikely((admin_queue->sq.tail & queue_size_mask) == 0))  in __ena_com_submit_admin_cmd()
254 		admin_queue->sq.phase = !admin_queue->sq.phase;  in __ena_com_submit_admin_cmd()
256 	ENA_DB_SYNC(&admin_queue->sq.mem_handle);  in __ena_com_submit_admin_cmd()
257 	ENA_REG_WRITE32(admin_queue->bus, admin_queue->sq.tail,  in __ena_com_submit_admin_cmd()
258 			admin_queue->sq.db_addr);  in __ena_com_submit_admin_cmd()
284 static struct ena_comp_ctx *ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,  in ena_com_submit_admin_cmd()  argument
293 	ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
294 	if (unlikely(!admin_queue->running_state)) {  in ena_com_submit_admin_cmd()
295 		ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
298 	comp_ctx = __ena_com_submit_admin_cmd(admin_queue, cmd,  in ena_com_submit_admin_cmd()
303 		admin_queue->running_state = false;  in ena_com_submit_admin_cmd()
304 	ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
444 static void ena_com_handle_single_admin_completion(struct ena_com_admin_queue *admin_queue,  in ena_com_handle_single_admin_completion()  argument
453 	comp_ctx = get_comp_ctxt(admin_queue, cmd_id, false);  in ena_com_handle_single_admin_completion()
456 		admin_queue->running_state = false;  in ena_com_handle_single_admin_completion()
466 	if (!admin_queue->polling)  in ena_com_handle_single_admin_completion()
470 static void ena_com_handle_admin_completion(struct ena_com_admin_queue *admin_queue)  in ena_com_handle_admin_completion()  argument
477 	head_masked = admin_queue->cq.head & (admin_queue->q_depth - 1);  in ena_com_handle_admin_completion()
478 	phase = admin_queue->cq.phase;  in ena_com_handle_admin_completion()
480 	cqe = &admin_queue->cq.entries[head_masked];  in ena_com_handle_admin_completion()
489 		ena_com_handle_single_admin_completion(admin_queue, cqe);  in ena_com_handle_admin_completion()
493 		if (unlikely(head_masked == admin_queue->q_depth)) {  in ena_com_handle_admin_completion()
498 		cqe = &admin_queue->cq.entries[head_masked];  in ena_com_handle_admin_completion()
501 	admin_queue->cq.head += comp_num;  in ena_com_handle_admin_completion()
502 	admin_queue->cq.phase = phase;  in ena_com_handle_admin_completion()
503 	admin_queue->sq.head += comp_num;  in ena_com_handle_admin_completion()
504 	admin_queue->stats.completed_cmd += comp_num;  in ena_com_handle_admin_completion()
539 						     struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq_polling()  argument
546 	timeout = ENA_GET_SYSTEM_TIMEOUT(admin_queue->completion_timeout);  in ena_com_wait_and_process_admin_cq_polling()
549 		ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
550 		ena_com_handle_admin_completion(admin_queue);  in ena_com_wait_and_process_admin_cq_polling()
551 		ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
559 			ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
560 			admin_queue->stats.no_completion++;  in ena_com_wait_and_process_admin_cq_polling()
561 			admin_queue->running_state = false;  in ena_com_wait_and_process_admin_cq_polling()
562 			ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
569 						 admin_queue->ena_dev->ena_min_poll_delay_us);  in ena_com_wait_and_process_admin_cq_polling()
574 		ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
575 		admin_queue->stats.aborted_cmd++;  in ena_com_wait_and_process_admin_cq_polling()
576 		ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
586 	comp_ctxt_release(admin_queue, comp_ctx);  in ena_com_wait_and_process_admin_cq_polling()
598 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_llq()  local
605 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_llq()
619 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_llq()
759 							struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq_interrupts()  argument
765 			    admin_queue->completion_timeout);  in ena_com_wait_and_process_admin_cq_interrupts()
773 		ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_interrupts()
774 		ena_com_handle_admin_completion(admin_queue);  in ena_com_wait_and_process_admin_cq_interrupts()
775 		admin_queue->stats.no_completion++;  in ena_com_wait_and_process_admin_cq_interrupts()
776 		ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_interrupts()
780 				    comp_ctx->cmd_opcode, admin_queue->auto_polling ? "ON" : "OFF");  in ena_com_wait_and_process_admin_cq_interrupts()
782 			if (admin_queue->auto_polling)  in ena_com_wait_and_process_admin_cq_interrupts()
783 				admin_queue->polling = true;  in ena_com_wait_and_process_admin_cq_interrupts()
792 		if (!admin_queue->polling) {  in ena_com_wait_and_process_admin_cq_interrupts()
793 			admin_queue->running_state = false;  in ena_com_wait_and_process_admin_cq_interrupts()
801 	comp_ctxt_release(admin_queue, comp_ctx);  in ena_com_wait_and_process_admin_cq_interrupts()
876 					     struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq()  argument
878 	if (admin_queue->polling)  in ena_com_wait_and_process_admin_cq()
880 								 admin_queue);  in ena_com_wait_and_process_admin_cq()
883 							    admin_queue);  in ena_com_wait_and_process_admin_cq()
889 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_destroy_io_sq()  local
909 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_destroy_io_sq()
1007 	struct ena_com_admin_queue *admin_queue;  in ena_com_get_feature_ex()  local
1017 	admin_queue = &ena_dev->admin_queue;  in ena_com_get_feature_ex()
1039 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_get_feature_ex()
1227 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_create_io_sq()  local
1269 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_create_io_sq()
1350 int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue,  in ena_com_execute_admin_command()  argument
1359 	comp_ctx = ena_com_submit_admin_cmd(admin_queue, cmd, cmd_size,  in ena_com_execute_admin_command()
1372 	ret = ena_com_wait_and_process_admin_cq(comp_ctx, admin_queue);  in ena_com_execute_admin_command()
1374 		if (admin_queue->running_state)  in ena_com_execute_admin_command()
1387 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_create_io_cq()  local
1412 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_create_io_cq()
1460 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_abort_admin_commands()  local
1464 	if (!admin_queue->comp_ctx)  in ena_com_abort_admin_commands()
1467 	for (i = 0; i < admin_queue->q_depth; i++) {  in ena_com_abort_admin_commands()
1468 		comp_ctx = get_comp_ctxt(admin_queue, i, false);  in ena_com_abort_admin_commands()
1480 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_wait_for_abort_completion()  local
1484 	ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1485 	while (ATOMIC32_READ(&admin_queue->outstanding_cmds) != 0) {  in ena_com_wait_for_abort_completion()
1486 		ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1488 		ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1490 	ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1496 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_destroy_io_cq()  local
1506 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_destroy_io_cq()
1520 	return ena_dev->admin_queue.running_state;  in ena_com_get_admin_running_state()
1525 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_admin_running_state()  local
1528 	ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_set_admin_running_state()
1529 	ena_dev->admin_queue.running_state = state;  in ena_com_set_admin_running_state()
1530 	ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_set_admin_running_state()
1547 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_aenq_config()  local
1567 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_aenq_config()
1574 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_aenq_config()
1660 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_admin_destroy()  local
1661 	struct ena_com_admin_cq *cq = &admin_queue->cq;  in ena_com_admin_destroy()
1662 	struct ena_com_admin_sq *sq = &admin_queue->sq;  in ena_com_admin_destroy()
1666 	if (admin_queue->comp_ctx) {  in ena_com_admin_destroy()
1667 		ENA_WAIT_EVENT_DESTROY(admin_queue->comp_ctx->wait_event);  in ena_com_admin_destroy()
1669 			     admin_queue->comp_ctx,  in ena_com_admin_destroy()
1670 			     (admin_queue->q_depth * sizeof(struct ena_comp_ctx)));  in ena_com_admin_destroy()
1673 	admin_queue->comp_ctx = NULL;  in ena_com_admin_destroy()
1674 	size = ADMIN_SQ_SIZE(admin_queue->q_depth);  in ena_com_admin_destroy()
1680 	size = ADMIN_CQ_SIZE(admin_queue->q_depth);  in ena_com_admin_destroy()
1691 	ENA_SPINLOCK_DESTROY(admin_queue->q_lock);  in ena_com_admin_destroy()
1703 	ena_dev->admin_queue.polling = polling;  in ena_com_set_admin_polling_mode()
1708 	return ena_dev->admin_queue.polling;  in ena_com_get_admin_polling_mode()
1714 	ena_dev->admin_queue.auto_polling = polling;  in ena_com_set_admin_auto_polling_mode()
1782 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_admin_init()  local
1798 	admin_queue->q_depth = ENA_ADMIN_QUEUE_DEPTH;  in ena_com_admin_init()
1800 	admin_queue->bus = ena_dev->bus;  in ena_com_admin_init()
1801 	admin_queue->q_dmadev = ena_dev->dmadev;  in ena_com_admin_init()
1802 	admin_queue->polling = false;  in ena_com_admin_init()
1803 	admin_queue->curr_cmd_id = 0;  in ena_com_admin_init()
1805 	ATOMIC32_SET(&admin_queue->outstanding_cmds, 0);  in ena_com_admin_init()
1807 	ENA_SPINLOCK_INIT(admin_queue->q_lock);  in ena_com_admin_init()
1809 	ret = ena_com_init_comp_ctxt(admin_queue);  in ena_com_admin_init()
1813 	ret = ena_com_admin_init_sq(admin_queue);  in ena_com_admin_init()
1817 	ret = ena_com_admin_init_cq(admin_queue);  in ena_com_admin_init()
1821 	admin_queue->sq.db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +  in ena_com_admin_init()
1824 	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->sq.dma_addr);  in ena_com_admin_init()
1825 	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->sq.dma_addr);  in ena_com_admin_init()
1830 	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->cq.dma_addr);  in ena_com_admin_init()
1831 	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->cq.dma_addr);  in ena_com_admin_init()
1837 	aq_caps |= admin_queue->q_depth & ENA_REGS_AQ_CAPS_AQ_DEPTH_MASK;  in ena_com_admin_init()
1843 	acq_caps |= admin_queue->q_depth & ENA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK;  in ena_com_admin_init()
1854 	admin_queue->ena_dev = ena_dev;  in ena_com_admin_init()
1855 	admin_queue->running_state = true;  in ena_com_admin_init()
2046 	ena_com_handle_admin_completion(&ena_dev->admin_queue);  in ena_com_admin_q_comp_intr_handler()
2184 		ena_dev->admin_queue.completion_timeout = timeout * 100000;  in ena_com_dev_reset()
2186 		ena_dev->admin_queue.completion_timeout = ADMIN_CMD_TIMEOUT_US;  in ena_com_dev_reset()
2197 	struct ena_com_admin_queue *admin_queue;  in ena_get_dev_stats()  local
2200 	admin_queue = &ena_dev->admin_queue;  in ena_get_dev_stats()
2206 	ret =  ena_com_execute_admin_command(admin_queue,  in ena_get_dev_stats()
2250 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_dev_mtu()  local
2261 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_dev_mtu()
2268 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_dev_mtu()
2300 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_hash_function()  local
2345 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_hash_function()
2480 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_hash_ctrl()  local
2513 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_hash_ctrl()
2640 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_indirect_table_set()  local
2679 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_indirect_table_set()
2831 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_host_attributes()  local
2842 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_host_attributes()
2865 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_host_attributes()