46 	struct amdgpu_mes *mes = &adev->mes;  in amdgpu_mes_kernel_doorbell_get()  local
53 	found = find_next_zero_bit(mes->doorbell_bitmap, mes->num_mes_dbs, offset);  in amdgpu_mes_kernel_doorbell_get()
54 	if (found >= mes->num_mes_dbs) {  in amdgpu_mes_kernel_doorbell_get()
59 	set_bit(found, mes->doorbell_bitmap);  in amdgpu_mes_kernel_doorbell_get()
62 	*doorbell_index = mes->db_start_dw_offset + found * 2;  in amdgpu_mes_kernel_doorbell_get()
70 	struct amdgpu_mes *mes = &adev->mes;  in amdgpu_mes_kernel_doorbell_free()  local
73 	rel_index = (doorbell_index - mes->db_start_dw_offset) / 2;  in amdgpu_mes_kernel_doorbell_free()
74 	old = test_and_clear_bit(rel_index, mes->doorbell_bitmap);  in amdgpu_mes_kernel_doorbell_free()
81 	struct amdgpu_mes *mes = &adev->mes;  in amdgpu_mes_doorbell_init()  local
84 	mes->doorbell_bitmap = bitmap_zalloc(PAGE_SIZE / sizeof(u32), GFP_KERNEL);  in amdgpu_mes_doorbell_init()
85 	if (!mes->doorbell_bitmap) {  in amdgpu_mes_doorbell_init()
90 	mes->num_mes_dbs = PAGE_SIZE / AMDGPU_ONE_DOORBELL_SIZE;  in amdgpu_mes_doorbell_init()
92 		adev->mes.aggregated_doorbells[i] = mes->db_start_dw_offset + i * 2;  in amdgpu_mes_doorbell_init()
93 		set_bit(i, mes->doorbell_bitmap);  in amdgpu_mes_doorbell_init()
106 	r = amdgpu_bo_create_kernel(adev, adev->mes.event_log_size, PAGE_SIZE,  in amdgpu_mes_event_log_init()
108 				    &adev->mes.event_log_gpu_obj,  in amdgpu_mes_event_log_init()
109 				    &adev->mes.event_log_gpu_addr,  in amdgpu_mes_event_log_init()
110 				    &adev->mes.event_log_cpu_addr);  in amdgpu_mes_event_log_init()
116 	memset(adev->mes.event_log_cpu_addr, 0, adev->mes.event_log_size);  in amdgpu_mes_event_log_init()
124 	bitmap_free(adev->mes.doorbell_bitmap);  in amdgpu_mes_doorbell_free()
131 	adev->mes.adev = adev;  in amdgpu_mes_init()
133 	idr_init(&adev->mes.pasid_idr);  in amdgpu_mes_init()
134 	idr_init(&adev->mes.gang_id_idr);  in amdgpu_mes_init()
135 	idr_init(&adev->mes.queue_id_idr);  in amdgpu_mes_init()
136 	ida_init(&adev->mes.doorbell_ida);  in amdgpu_mes_init()
137 	spin_lock_init(&adev->mes.queue_id_lock);  in amdgpu_mes_init()
138 	mutex_init(&adev->mes.mutex_hidden);  in amdgpu_mes_init()
141 		spin_lock_init(&adev->mes.ring_lock[i]);  in amdgpu_mes_init()
143 	adev->mes.total_max_queue = AMDGPU_FENCE_MES_QUEUE_ID_MASK;  in amdgpu_mes_init()
144 	adev->mes.vmid_mask_mmhub = 0xffffff00;  in amdgpu_mes_init()
145 	adev->mes.vmid_mask_gfxhub = 0xffffff00;  in amdgpu_mes_init()
150 		adev->mes.compute_hqd_mask[i] = 0xc;  in amdgpu_mes_init()
154 		adev->mes.gfx_hqd_mask[i] = i ? 0 : 0xfffffffe;  in amdgpu_mes_init()
159 		adev->mes.sdma_hqd_mask[i] = 0xfc;  in amdgpu_mes_init()
163 		r = amdgpu_device_wb_get(adev, &adev->mes.sch_ctx_offs[i]);  in amdgpu_mes_init()
170 		adev->mes.sch_ctx_gpu_addr[i] =  in amdgpu_mes_init()
171 			adev->wb.gpu_addr + (adev->mes.sch_ctx_offs[i] * 4);  in amdgpu_mes_init()
172 		adev->mes.sch_ctx_ptr[i] =  in amdgpu_mes_init()
173 			(uint64_t *)&adev->wb.wb[adev->mes.sch_ctx_offs[i]];  in amdgpu_mes_init()
176 				 &adev->mes.query_status_fence_offs[i]);  in amdgpu_mes_init()
183 		adev->mes.query_status_fence_gpu_addr[i] = adev->wb.gpu_addr +  in amdgpu_mes_init()
184 			(adev->mes.query_status_fence_offs[i] * 4);  in amdgpu_mes_init()
185 		adev->mes.query_status_fence_ptr[i] =  in amdgpu_mes_init()
186 			(uint64_t *)&adev->wb.wb[adev->mes.query_status_fence_offs[i]];  in amdgpu_mes_init()
203 		if (adev->mes.sch_ctx_ptr[i])  in amdgpu_mes_init()
204 			amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs[i]);  in amdgpu_mes_init()
205 		if (adev->mes.query_status_fence_ptr[i])  in amdgpu_mes_init()
207 				      adev->mes.query_status_fence_offs[i]);  in amdgpu_mes_init()
210 	idr_destroy(&adev->mes.pasid_idr);  in amdgpu_mes_init()
211 	idr_destroy(&adev->mes.gang_id_idr);  in amdgpu_mes_init()
212 	idr_destroy(&adev->mes.queue_id_idr);  in amdgpu_mes_init()
213 	ida_destroy(&adev->mes.doorbell_ida);  in amdgpu_mes_init()
214 	mutex_destroy(&adev->mes.mutex_hidden);  in amdgpu_mes_init()
222 	amdgpu_bo_free_kernel(&adev->mes.event_log_gpu_obj,  in amdgpu_mes_fini()
223 			      &adev->mes.event_log_gpu_addr,  in amdgpu_mes_fini()
224 			      &adev->mes.event_log_cpu_addr);  in amdgpu_mes_fini()
227 		if (adev->mes.sch_ctx_ptr[i])  in amdgpu_mes_fini()
228 			amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs[i]);  in amdgpu_mes_fini()
229 		if (adev->mes.query_status_fence_ptr[i])  in amdgpu_mes_fini()
231 				      adev->mes.query_status_fence_offs[i]);  in amdgpu_mes_fini()
236 	idr_destroy(&adev->mes.pasid_idr);  in amdgpu_mes_fini()
237 	idr_destroy(&adev->mes.gang_id_idr);  in amdgpu_mes_fini()
238 	idr_destroy(&adev->mes.queue_id_idr);  in amdgpu_mes_fini()
239 	ida_destroy(&adev->mes.doorbell_ida);  in amdgpu_mes_fini()
240 	mutex_destroy(&adev->mes.mutex_hidden);  in amdgpu_mes_fini()
279 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_create_process()
282 	r = idr_alloc(&adev->mes.pasid_idr, process, pasid, pasid + 1,  in amdgpu_mes_create_process()
292 	process->process_quantum = adev->mes.default_process_quantum;  in amdgpu_mes_create_process()
295 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_create_process()
299 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_create_process()
321 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_destroy_process()
323 	process = idr_find(&adev->mes.pasid_idr, pasid);  in amdgpu_mes_destroy_process()
326 		amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_destroy_process()
333 			spin_lock_irqsave(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_destroy_process()
334 			idr_remove(&adev->mes.queue_id_idr, queue->queue_id);  in amdgpu_mes_destroy_process()
335 			spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_destroy_process()
340 			r = adev->mes.funcs->remove_hw_queue(&adev->mes,  in amdgpu_mes_destroy_process()
346 		idr_remove(&adev->mes.gang_id_idr, gang->gang_id);  in amdgpu_mes_destroy_process()
349 	idr_remove(&adev->mes.pasid_idr, pasid);  in amdgpu_mes_destroy_process()
350 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_destroy_process()
403 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_add_gang()
405 	process = idr_find(&adev->mes.pasid_idr, pasid);  in amdgpu_mes_add_gang()
413 	r = idr_alloc(&adev->mes.gang_id_idr, gang, 1, 0,  in amdgpu_mes_add_gang()
427 		gprops->gang_quantum : adev->mes.default_gang_quantum;  in amdgpu_mes_add_gang()
432 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_add_gang()
436 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_add_gang()
453 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_remove_gang()
455 	gang = idr_find(&adev->mes.gang_id_idr, gang_id);  in amdgpu_mes_remove_gang()
458 		amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_remove_gang()
464 		amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_remove_gang()
468 	idr_remove(&adev->mes.gang_id_idr, gang->gang_id);  in amdgpu_mes_remove_gang()
470 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_remove_gang()
496 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_suspend()
497 	r = adev->mes.funcs->suspend_gang(&adev->mes, &input);  in amdgpu_mes_suspend()
498 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_suspend()
520 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_resume()
521 	r = adev->mes.funcs->resume_gang(&adev->mes, &input);  in amdgpu_mes_resume()
522 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_resume()
624 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_add_hw_queue()
626 	gang = idr_find(&adev->mes.gang_id_idr, gang_id);  in amdgpu_mes_add_hw_queue()
634 	spin_lock_irqsave(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_add_hw_queue()
635 	r = idr_alloc(&adev->mes.queue_id_idr, queue, 1, 0,  in amdgpu_mes_add_hw_queue()
638 		spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_add_hw_queue()
641 	spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_add_hw_queue()
678 	r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input);  in amdgpu_mes_add_hw_queue()
699 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_add_hw_queue()
705 	spin_lock_irqsave(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_add_hw_queue()
706 	idr_remove(&adev->mes.queue_id_idr, queue->queue_id);  in amdgpu_mes_add_hw_queue()
707 	spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_add_hw_queue()
709 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_add_hw_queue()
728 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_remove_hw_queue()
731 	spin_lock_irqsave(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_remove_hw_queue()
733 	queue = idr_find(&adev->mes.queue_id_idr, queue_id);  in amdgpu_mes_remove_hw_queue()
735 		spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_remove_hw_queue()
736 		amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_remove_hw_queue()
741 	idr_remove(&adev->mes.queue_id_idr, queue_id);  in amdgpu_mes_remove_hw_queue()
742 	spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_remove_hw_queue()
751 	r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input);  in amdgpu_mes_remove_hw_queue()
758 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_remove_hw_queue()
777 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_reset_hw_queue()
780 	spin_lock_irqsave(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_reset_hw_queue()
782 	queue = idr_find(&adev->mes.queue_id_idr, queue_id);  in amdgpu_mes_reset_hw_queue()
784 		spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_reset_hw_queue()
785 		amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_reset_hw_queue()
789 	spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags);  in amdgpu_mes_reset_hw_queue()
798 	r = adev->mes.funcs->reset_hw_queue(&adev->mes, &queue_input);  in amdgpu_mes_reset_hw_queue()
803 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_reset_hw_queue()
820 	r = adev->mes.funcs->reset_hw_queue(&adev->mes, &queue_input);  in amdgpu_mes_reset_hw_queue_mmio()
842 	r = adev->mes.funcs->map_legacy_queue(&adev->mes, &queue_input);  in amdgpu_mes_map_legacy_queue()
865 	r = adev->mes.funcs->unmap_legacy_queue(&adev->mes, &queue_input);  in amdgpu_mes_unmap_legacy_queue()
892 	r = adev->mes.funcs->reset_legacy_queue(&adev->mes, &queue_input);  in amdgpu_mes_reset_legacy_queue()
917 	if (!adev->mes.funcs->misc_op) {  in amdgpu_mes_rreg()
922 	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);  in amdgpu_mes_rreg()
944 	if (!adev->mes.funcs->misc_op) {  in amdgpu_mes_wreg()
950 	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);  in amdgpu_mes_wreg()
971 	if (!adev->mes.funcs->misc_op) {  in amdgpu_mes_reg_write_reg_wait()
977 	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);  in amdgpu_mes_reg_write_reg_wait()
996 	if (!adev->mes.funcs->misc_op) {  in amdgpu_mes_reg_wait()
1002 	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);  in amdgpu_mes_reg_wait()
1020 	if (!adev->mes.funcs->misc_op) {  in amdgpu_mes_set_shader_debugger()
1037 	if (((adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK) >>  in amdgpu_mes_set_shader_debugger()
1041 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_set_shader_debugger()
1043 	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);  in amdgpu_mes_set_shader_debugger()
1047 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_set_shader_debugger()
1058 	if (!adev->mes.funcs->misc_op) {  in amdgpu_mes_flush_shader_debugger()
1067 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_flush_shader_debugger()
1069 	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);  in amdgpu_mes_flush_shader_debugger()
1073 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_flush_shader_debugger()
1147 	amdgpu_mes_lock(&adev->mes);  in amdgpu_mes_add_ring()
1148 	gang = idr_find(&adev->mes.gang_id_idr, gang_id);  in amdgpu_mes_add_ring()
1151 		amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_add_ring()
1158 		amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_add_ring()
1197 		amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_add_ring()
1205 	amdgpu_mes_unlock(&adev->mes);  in amdgpu_mes_add_ring()
1250 	return adev->mes.aggregated_doorbells[prio];  in amdgpu_mes_get_aggregated_doorbell_index()
1423 	gprops.gang_quantum = adev->mes.default_gang_quantum;  in amdgpu_mes_test_create_gang_and_queues()
1607 	r = amdgpu_ucode_request(adev, &adev->mes.fw[pipe], AMDGPU_UCODE_REQUIRED,  in amdgpu_mes_init_microcode()
1611 		r = amdgpu_ucode_request(adev, &adev->mes.fw[pipe],  in amdgpu_mes_init_microcode()
1620 		adev->mes.fw[pipe]->data;  in amdgpu_mes_init_microcode()
1621 	adev->mes.uc_start_addr[pipe] =  in amdgpu_mes_init_microcode()
1624 	adev->mes.data_start_addr[pipe] =  in amdgpu_mes_init_microcode()
1627 	ucode_ptr = (u32 *)(adev->mes.fw[pipe]->data +  in amdgpu_mes_init_microcode()
1629 	adev->mes.fw_version[pipe] =  in amdgpu_mes_init_microcode()
1645 		info->fw = adev->mes.fw[pipe];  in amdgpu_mes_init_microcode()
1652 		info->fw = adev->mes.fw[pipe];  in amdgpu_mes_init_microcode()
1660 	amdgpu_ucode_release(&adev->mes.fw[pipe]);  in amdgpu_mes_init_microcode()
1666 	uint32_t mes_rev = adev->mes.sched_version & AMDGPU_MES_VERSION_MASK;  in amdgpu_mes_suspend_resume_all_supported()
1687 	if (!adev->mes.funcs->misc_op) {  in amdgpu_mes_set_enforce_isolation()
1693 	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);  in amdgpu_mes_set_enforce_isolation()
1723 	uint32_t *mem = (uint32_t *)(adev->mes.event_log_cpu_addr);  in amdgpu_debugfs_mes_event_log_show()
1726 		     mem, adev->mes.event_log_size, false);  in amdgpu_debugfs_mes_event_log_show()