83 static int ena_com_admin_init_sq(struct ena_com_admin_queue *admin_queue)  in ena_com_admin_init_sq()  argument
85 	struct ena_com_dev *ena_dev = admin_queue->ena_dev;  in ena_com_admin_init_sq()
86 	struct ena_com_admin_sq *sq = &admin_queue->sq;  in ena_com_admin_init_sq()
87 	u16 size = ADMIN_SQ_SIZE(admin_queue->q_depth);  in ena_com_admin_init_sq()
89 	ENA_MEM_ALLOC_COHERENT(admin_queue->q_dmadev, size, sq->entries, sq->dma_addr,  in ena_com_admin_init_sq()
106 static int ena_com_admin_init_cq(struct ena_com_admin_queue *admin_queue)  in ena_com_admin_init_cq()  argument
108 	struct ena_com_dev *ena_dev = admin_queue->ena_dev;  in ena_com_admin_init_cq()
109 	struct ena_com_admin_cq *cq = &admin_queue->cq;  in ena_com_admin_init_cq()
110 	u16 size = ADMIN_CQ_SIZE(admin_queue->q_depth);  in ena_com_admin_init_cq()
112 	ENA_MEM_ALLOC_COHERENT(admin_queue->q_dmadev, size, cq->entries, cq->dma_addr,  in ena_com_admin_init_cq()
178 static struct ena_comp_ctx *get_comp_ctxt(struct ena_com_admin_queue *admin_queue,  in get_comp_ctxt()  argument
181 	if (unlikely(command_id >= admin_queue->q_depth)) {  in get_comp_ctxt()
182 		ena_trc_err(admin_queue->ena_dev,  in get_comp_ctxt()
184 			    command_id, admin_queue->q_depth);  in get_comp_ctxt()
188 	if (unlikely(!admin_queue->comp_ctx)) {  in get_comp_ctxt()
189 		ena_trc_err(admin_queue->ena_dev,  in get_comp_ctxt()
194 	if (unlikely(admin_queue->comp_ctx[command_id].occupied && capture)) {  in get_comp_ctxt()
195 		ena_trc_err(admin_queue->ena_dev,  in get_comp_ctxt()
201 		ATOMIC32_INC(&admin_queue->outstanding_cmds);  in get_comp_ctxt()
202 		admin_queue->comp_ctx[command_id].occupied = true;  in get_comp_ctxt()
205 	return &admin_queue->comp_ctx[command_id];  in get_comp_ctxt()
208 static struct ena_comp_ctx *__ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,  in __ena_com_submit_admin_cmd()  argument
219 	queue_size_mask = admin_queue->q_depth - 1;  in __ena_com_submit_admin_cmd()
221 	tail_masked = admin_queue->sq.tail & queue_size_mask;  in __ena_com_submit_admin_cmd()
224 	cnt = (u16)ATOMIC32_READ(&admin_queue->outstanding_cmds);  in __ena_com_submit_admin_cmd()
225 	if (cnt >= admin_queue->q_depth) {  in __ena_com_submit_admin_cmd()
226 		ena_trc_dbg(admin_queue->ena_dev, "Admin queue is full.\n");  in __ena_com_submit_admin_cmd()
227 		admin_queue->stats.out_of_space++;  in __ena_com_submit_admin_cmd()
231 	cmd_id = admin_queue->curr_cmd_id;  in __ena_com_submit_admin_cmd()
233 	cmd->aq_common_descriptor.flags |= admin_queue->sq.phase &  in __ena_com_submit_admin_cmd()
239 	comp_ctx = get_comp_ctxt(admin_queue, cmd_id, true);  in __ena_com_submit_admin_cmd()
250 	memcpy(&admin_queue->sq.entries[tail_masked], cmd, cmd_size_in_bytes);  in __ena_com_submit_admin_cmd()
252 	admin_queue->curr_cmd_id = (admin_queue->curr_cmd_id + 1) &  in __ena_com_submit_admin_cmd()
255 	admin_queue->sq.tail++;  in __ena_com_submit_admin_cmd()
256 	admin_queue->stats.submitted_cmd++;  in __ena_com_submit_admin_cmd()
258 	if (unlikely((admin_queue->sq.tail & queue_size_mask) == 0))  in __ena_com_submit_admin_cmd()
259 		admin_queue->sq.phase = !admin_queue->sq.phase;  in __ena_com_submit_admin_cmd()
261 	ENA_DB_SYNC(&admin_queue->sq.mem_handle);  in __ena_com_submit_admin_cmd()
262 	ENA_REG_WRITE32(admin_queue->bus, admin_queue->sq.tail,  in __ena_com_submit_admin_cmd()
263 			admin_queue->sq.db_addr);  in __ena_com_submit_admin_cmd()
268 static int ena_com_init_comp_ctxt(struct ena_com_admin_queue *admin_queue)  in ena_com_init_comp_ctxt()  argument
270 	struct ena_com_dev *ena_dev = admin_queue->ena_dev;  in ena_com_init_comp_ctxt()
271 	size_t size = admin_queue->q_depth * sizeof(struct ena_comp_ctx);  in ena_com_init_comp_ctxt()
275 	admin_queue->comp_ctx = ENA_MEM_ALLOC(admin_queue->q_dmadev, size);  in ena_com_init_comp_ctxt()
276 	if (unlikely(!admin_queue->comp_ctx)) {  in ena_com_init_comp_ctxt()
281 	for (i = 0; i < admin_queue->q_depth; i++) {  in ena_com_init_comp_ctxt()
282 		comp_ctx = get_comp_ctxt(admin_queue, i, false);  in ena_com_init_comp_ctxt()
290 static struct ena_comp_ctx *ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,  in ena_com_submit_admin_cmd()  argument
299 	ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
300 	if (unlikely(!admin_queue->running_state)) {  in ena_com_submit_admin_cmd()
301 		ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
304 	comp_ctx = __ena_com_submit_admin_cmd(admin_queue, cmd,  in ena_com_submit_admin_cmd()
309 		admin_queue->running_state = false;  in ena_com_submit_admin_cmd()
310 	ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
450 static void ena_com_handle_single_admin_completion(struct ena_com_admin_queue *admin_queue,  in ena_com_handle_single_admin_completion()  argument
459 	comp_ctx = get_comp_ctxt(admin_queue, cmd_id, false);  in ena_com_handle_single_admin_completion()
461 		ena_trc_err(admin_queue->ena_dev,  in ena_com_handle_single_admin_completion()
463 		admin_queue->running_state = false;  in ena_com_handle_single_admin_completion()
473 	if (!admin_queue->polling)  in ena_com_handle_single_admin_completion()
477 static void ena_com_handle_admin_completion(struct ena_com_admin_queue *admin_queue)  in ena_com_handle_admin_completion()  argument
484 	head_masked = admin_queue->cq.head & (admin_queue->q_depth - 1);  in ena_com_handle_admin_completion()
485 	phase = admin_queue->cq.phase;  in ena_com_handle_admin_completion()
487 	cqe = &admin_queue->cq.entries[head_masked];  in ena_com_handle_admin_completion()
496 		ena_com_handle_single_admin_completion(admin_queue, cqe);  in ena_com_handle_admin_completion()
500 		if (unlikely(head_masked == admin_queue->q_depth)) {  in ena_com_handle_admin_completion()
505 		cqe = &admin_queue->cq.entries[head_masked];  in ena_com_handle_admin_completion()
508 	admin_queue->cq.head += comp_num;  in ena_com_handle_admin_completion()
509 	admin_queue->cq.phase = phase;  in ena_com_handle_admin_completion()
510 	admin_queue->sq.head += comp_num;  in ena_com_handle_admin_completion()
511 	admin_queue->stats.completed_cmd += comp_num;  in ena_com_handle_admin_completion()
514 static int ena_com_comp_status_to_errno(struct ena_com_admin_queue *admin_queue,  in ena_com_comp_status_to_errno()  argument
518 		ena_trc_err(admin_queue->ena_dev,  in ena_com_comp_status_to_errno()
548 						     struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq_polling()  argument
555 	timeout = ENA_GET_SYSTEM_TIMEOUT(admin_queue->completion_timeout);  in ena_com_wait_and_process_admin_cq_polling()
558 		ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
559 		ena_com_handle_admin_completion(admin_queue);  in ena_com_wait_and_process_admin_cq_polling()
560 		ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
566 			ena_trc_err(admin_queue->ena_dev,  in ena_com_wait_and_process_admin_cq_polling()
569 			ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
570 			admin_queue->stats.no_completion++;  in ena_com_wait_and_process_admin_cq_polling()
571 			admin_queue->running_state = false;  in ena_com_wait_and_process_admin_cq_polling()
572 			ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
579 						 admin_queue->ena_dev->ena_min_poll_delay_us);  in ena_com_wait_and_process_admin_cq_polling()
583 		ena_trc_err(admin_queue->ena_dev, "Command was aborted\n");  in ena_com_wait_and_process_admin_cq_polling()
584 		ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
585 		admin_queue->stats.aborted_cmd++;  in ena_com_wait_and_process_admin_cq_polling()
586 		ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
592 		 admin_queue->ena_dev, "Invalid comp status %d\n",  in ena_com_wait_and_process_admin_cq_polling()
595 	ret = ena_com_comp_status_to_errno(admin_queue, comp_ctx->comp_status);  in ena_com_wait_and_process_admin_cq_polling()
597 	comp_ctxt_release(admin_queue, comp_ctx);  in ena_com_wait_and_process_admin_cq_polling()
609 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_llq()  local
616 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_llq()
630 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_llq()
771 							struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq_interrupts()  argument
777 			    admin_queue->completion_timeout);  in ena_com_wait_and_process_admin_cq_interrupts()
785 		ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_interrupts()
786 		ena_com_handle_admin_completion(admin_queue);  in ena_com_wait_and_process_admin_cq_interrupts()
787 		admin_queue->stats.no_completion++;  in ena_com_wait_and_process_admin_cq_interrupts()
788 		ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_interrupts()
791 			ena_trc_err(admin_queue->ena_dev,  in ena_com_wait_and_process_admin_cq_interrupts()
793 				    comp_ctx->cmd_opcode, admin_queue->auto_polling ? "ON" : "OFF");  in ena_com_wait_and_process_admin_cq_interrupts()
795 			if (admin_queue->auto_polling)  in ena_com_wait_and_process_admin_cq_interrupts()
796 				admin_queue->polling = true;  in ena_com_wait_and_process_admin_cq_interrupts()
798 			ena_trc_err(admin_queue->ena_dev,  in ena_com_wait_and_process_admin_cq_interrupts()
806 		if (!admin_queue->polling) {  in ena_com_wait_and_process_admin_cq_interrupts()
807 			admin_queue->running_state = false;  in ena_com_wait_and_process_admin_cq_interrupts()
813 	ret = ena_com_comp_status_to_errno(admin_queue, comp_ctx->comp_status);  in ena_com_wait_and_process_admin_cq_interrupts()
815 	comp_ctxt_release(admin_queue, comp_ctx);  in ena_com_wait_and_process_admin_cq_interrupts()
890 					     struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq()  argument
892 	if (admin_queue->polling)  in ena_com_wait_and_process_admin_cq()
894 								 admin_queue);  in ena_com_wait_and_process_admin_cq()
897 							    admin_queue);  in ena_com_wait_and_process_admin_cq()
903 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_destroy_io_sq()  local
923 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_destroy_io_sq()
1021 	struct ena_com_admin_queue *admin_queue;  in ena_com_get_feature_ex()  local
1031 	admin_queue = &ena_dev->admin_queue;  in ena_com_get_feature_ex()
1053 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_get_feature_ex()
1241 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_create_io_sq()  local
1283 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_create_io_sq()
1364 int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue,  in ena_com_execute_admin_command()  argument
1373 	comp_ctx = ena_com_submit_admin_cmd(admin_queue, cmd, cmd_size,  in ena_com_execute_admin_command()
1377 			ena_trc_dbg(admin_queue->ena_dev,  in ena_com_execute_admin_command()
1381 			ena_trc_err(admin_queue->ena_dev,  in ena_com_execute_admin_command()
1388 	ret = ena_com_wait_and_process_admin_cq(comp_ctx, admin_queue);  in ena_com_execute_admin_command()
1390 		if (admin_queue->running_state)  in ena_com_execute_admin_command()
1391 			ena_trc_err(admin_queue->ena_dev,  in ena_com_execute_admin_command()
1394 			ena_trc_dbg(admin_queue->ena_dev,  in ena_com_execute_admin_command()
1403 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_create_io_cq()  local
1428 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_create_io_cq()
1476 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_abort_admin_commands()  local
1480 	if (!admin_queue->comp_ctx)  in ena_com_abort_admin_commands()
1483 	for (i = 0; i < admin_queue->q_depth; i++) {  in ena_com_abort_admin_commands()
1484 		comp_ctx = get_comp_ctxt(admin_queue, i, false);  in ena_com_abort_admin_commands()
1496 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_wait_for_abort_completion()  local
1500 	ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1501 	while (ATOMIC32_READ(&admin_queue->outstanding_cmds) != 0) {  in ena_com_wait_for_abort_completion()
1502 		ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1504 		ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1506 	ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1512 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_destroy_io_cq()  local
1522 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_destroy_io_cq()
1536 	return ena_dev->admin_queue.running_state;  in ena_com_get_admin_running_state()
1541 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_admin_running_state()  local
1544 	ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags);  in ena_com_set_admin_running_state()
1545 	ena_dev->admin_queue.running_state = state;  in ena_com_set_admin_running_state()
1546 	ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);  in ena_com_set_admin_running_state()
1563 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_aenq_config()  local
1583 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_aenq_config()
1590 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_aenq_config()
1676 				      struct ena_com_admin_queue *admin_queue)  in ena_com_free_ena_admin_queue_comp_ctx()  argument
1679 	if (!admin_queue->comp_ctx)  in ena_com_free_ena_admin_queue_comp_ctx()
1682 	ENA_WAIT_EVENTS_DESTROY(admin_queue);  in ena_com_free_ena_admin_queue_comp_ctx()
1684 		     admin_queue->comp_ctx,  in ena_com_free_ena_admin_queue_comp_ctx()
1685 		     (admin_queue->q_depth * sizeof(struct ena_comp_ctx)));  in ena_com_free_ena_admin_queue_comp_ctx()
1687 	admin_queue->comp_ctx = NULL;  in ena_com_free_ena_admin_queue_comp_ctx()
1692 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_admin_destroy()  local
1693 	struct ena_com_admin_cq *cq = &admin_queue->cq;  in ena_com_admin_destroy()
1694 	struct ena_com_admin_sq *sq = &admin_queue->sq;  in ena_com_admin_destroy()
1698 	ena_com_free_ena_admin_queue_comp_ctx(ena_dev, admin_queue);  in ena_com_admin_destroy()
1700 	size = ADMIN_SQ_SIZE(admin_queue->q_depth);  in ena_com_admin_destroy()
1706 	size = ADMIN_CQ_SIZE(admin_queue->q_depth);  in ena_com_admin_destroy()
1717 	ENA_SPINLOCK_DESTROY(admin_queue->q_lock);  in ena_com_admin_destroy()
1729 	ena_dev->admin_queue.polling = polling;  in ena_com_set_admin_polling_mode()
1734 	return ena_dev->admin_queue.polling;  in ena_com_get_admin_polling_mode()
1740 	ena_dev->admin_queue.auto_polling = polling;  in ena_com_set_admin_auto_polling_mode()
1808 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_admin_init()  local
1824 	admin_queue->q_depth = ENA_ADMIN_QUEUE_DEPTH;  in ena_com_admin_init()
1826 	admin_queue->bus = ena_dev->bus;  in ena_com_admin_init()
1827 	admin_queue->q_dmadev = ena_dev->dmadev;  in ena_com_admin_init()
1828 	admin_queue->polling = false;  in ena_com_admin_init()
1829 	admin_queue->curr_cmd_id = 0;  in ena_com_admin_init()
1831 	ATOMIC32_SET(&admin_queue->outstanding_cmds, 0);  in ena_com_admin_init()
1833 	ENA_SPINLOCK_INIT(admin_queue->q_lock);  in ena_com_admin_init()
1835 	ret = ena_com_init_comp_ctxt(admin_queue);  in ena_com_admin_init()
1839 	ret = ena_com_admin_init_sq(admin_queue);  in ena_com_admin_init()
1843 	ret = ena_com_admin_init_cq(admin_queue);  in ena_com_admin_init()
1847 	admin_queue->sq.db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +  in ena_com_admin_init()
1850 	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->sq.dma_addr);  in ena_com_admin_init()
1851 	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->sq.dma_addr);  in ena_com_admin_init()
1856 	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->cq.dma_addr);  in ena_com_admin_init()
1857 	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->cq.dma_addr);  in ena_com_admin_init()
1863 	aq_caps |= admin_queue->q_depth & ENA_REGS_AQ_CAPS_AQ_DEPTH_MASK;  in ena_com_admin_init()
1869 	acq_caps |= admin_queue->q_depth & ENA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK;  in ena_com_admin_init()
1880 	admin_queue->ena_dev = ena_dev;  in ena_com_admin_init()
1881 	admin_queue->running_state = true;  in ena_com_admin_init()
2062 	ena_com_handle_admin_completion(&ena_dev->admin_queue);  in ena_com_admin_q_comp_intr_handler()
2200 		ena_dev->admin_queue.completion_timeout = timeout * 100000;  in ena_com_dev_reset()
2202 		ena_dev->admin_queue.completion_timeout = ADMIN_CMD_TIMEOUT_US;  in ena_com_dev_reset()
2213 	struct ena_com_admin_queue *admin_queue;  in ena_get_dev_stats()  local
2216 	admin_queue = &ena_dev->admin_queue;  in ena_get_dev_stats()
2222 	ret =  ena_com_execute_admin_command(admin_queue,  in ena_get_dev_stats()
2266 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_dev_mtu()  local
2277 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_dev_mtu()
2284 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_dev_mtu()
2316 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_hash_function()  local
2361 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_hash_function()
2497 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_hash_ctrl()  local
2530 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_hash_ctrl()
2657 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_indirect_table_set()  local
2696 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_indirect_table_set()
2848 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_host_attributes()  local
2859 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_host_attributes()
2882 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_host_attributes()