83 void kfd_process_dequeue_from_device(struct kfd_process_device *pdd)  in kfd_process_dequeue_from_device()  argument
85 	struct kfd_node *dev = pdd->dev;  in kfd_process_dequeue_from_device()
87 	if (pdd->already_dequeued)  in kfd_process_dequeue_from_device()
93 	dev->dqm->ops.process_termination(dev->dqm, &pdd->qpd);  in kfd_process_dequeue_from_device()
94 	if (dev->kfd->shared_resources.enable_mes && !!pdd->proc_ctx_gpu_addr &&  in kfd_process_dequeue_from_device()
97 						 pdd->proc_ctx_gpu_addr);  in kfd_process_dequeue_from_device()
100 	pdd->already_dequeued = true;  in kfd_process_dequeue_from_device()
109 	struct kfd_process_device *pdd;  in pqm_set_gws()  local
124 	pdd = kfd_get_process_device_data(dev, pqm->process);  in pqm_set_gws()
125 	if (!pdd) {  in pqm_set_gws()
131 	if (gws && pdd->qpd.num_gws)  in pqm_set_gws()
134 	if (!gws && pdd->qpd.num_gws == 0)  in pqm_set_gws()
142 			ret = amdgpu_amdkfd_add_gws_to_process(pdd->process->kgd_process_info,  in pqm_set_gws()
145 			ret = amdgpu_amdkfd_remove_gws_from_process(pdd->process->kgd_process_info,  in pqm_set_gws()
160 	pdd->qpd.num_gws = gws ? dev->adev->gds.gws_size : 0;  in pqm_set_gws()
191 	struct kfd_process_device *pdd;  in pqm_clean_queue_resource()  local
195 	pdd = kfd_get_process_device_data(dev, pqm->process);  in pqm_clean_queue_resource()
196 	if (!pdd) {  in pqm_clean_queue_resource()
208 		pdd->qpd.num_gws = 0;  in pqm_clean_queue_resource()
223 			struct kfd_process_device *pdd = kfd_get_process_device_data(pqn->q->device,  in pqm_uninit()  local
225 			if (pdd) {  in pqm_uninit()
226 				kfd_queue_unref_bo_vas(pdd, &pqn->q->properties);  in pqm_uninit()
227 				kfd_queue_release_buffers(pdd, &pqn->q->properties);  in pqm_uninit()
229 				WARN_ON(!pdd);  in pqm_uninit()
320 	struct kfd_process_device *pdd;  in pqm_create_queue()  local
339 	pdd = kfd_get_process_device_data(dev, pqm->process);  in pqm_create_queue()
340 	if (!pdd) {  in pqm_create_queue()
351 	if ((pdd->qpd.is_debug) || (type == KFD_QUEUE_TYPE_DIQ))  in pqm_create_queue()
354 	if (pdd->qpd.queue_count >= max_queues)  in pqm_create_queue()
367 	if (list_empty(&pdd->qpd.queues_list) &&  in pqm_create_queue()
368 	    list_empty(&pdd->qpd.priv_queue_list))  in pqm_create_queue()
369 		dev->dqm->ops.register_process(dev->dqm, &pdd->qpd);  in pqm_create_queue()
372 	if (!pdd->proc_ctx_cpu_ptr && dev->kfd->shared_resources.enable_mes) {  in pqm_create_queue()
375 						     &pdd->proc_ctx_bo,  in pqm_create_queue()
376 						     &pdd->proc_ctx_gpu_addr,  in pqm_create_queue()
377 						     &pdd->proc_ctx_cpu_ptr,  in pqm_create_queue()
383 		memset(pdd->proc_ctx_cpu_ptr, 0, AMDGPU_MES_PROC_CTX_SIZE);  in pqm_create_queue()
407 		retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data,  in pqm_create_queue()
428 		retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data,  in pqm_create_queue()
441 		retval = kfd_process_drain_interrupts(pdd);  in pqm_create_queue()
446 							kq, &pdd->qpd);  in pqm_create_queue()
466 		uint32_t first_db_index = amdgpu_doorbell_index_on_bar(pdd->dev->adev,  in pqm_create_queue()
467 								       pdd->qpd.proc_doorbells,  in pqm_create_queue()
469 								       pdd->dev->kfd->device_info.doorbell_size);  in pqm_create_queue()
495 	if (list_empty(&pdd->qpd.queues_list) &&  in pqm_create_queue()
496 	    list_empty(&pdd->qpd.priv_queue_list))  in pqm_create_queue()
497 		dev->dqm->ops.unregister_process(dev->dqm, &pdd->qpd);  in pqm_create_queue()
504 	struct kfd_process_device *pdd;  in pqm_destroy_queue()  local
527 	pdd = kfd_get_process_device_data(dev, pqm->process);  in pqm_destroy_queue()
528 	if (!pdd) {  in pqm_destroy_queue()
536 		dqm->ops.destroy_kernel_queue(dqm, pqn->kq, &pdd->qpd);  in pqm_destroy_queue()
541 		retval = kfd_queue_unref_bo_vas(pdd, &pqn->q->properties);  in pqm_destroy_queue()
546 		retval = dqm->ops.destroy_queue(dqm, &pdd->qpd, pqn->q);  in pqm_destroy_queue()
549 				pdd->pasid,  in pqm_destroy_queue()
555 		kfd_queue_release_buffers(pdd, &pqn->q->properties);  in pqm_destroy_queue()
564 	if (list_empty(&pdd->qpd.queues_list) &&  in pqm_destroy_queue()
565 	    list_empty(&pdd->qpd.priv_queue_list))  in pqm_destroy_queue()
566 		dqm->ops.unregister_process(dqm, &pdd->qpd);  in pqm_destroy_queue()
588 		struct kfd_process_device *pdd;  in pqm_update_queue_properties()  local
593 		pdd = kfd_get_process_device_data(q->device, q->process);  in pqm_update_queue_properties()
594 		if (!pdd)  in pqm_update_queue_properties()
596 		vm = drm_priv_to_vm(pdd->drm_priv);  in pqm_update_queue_properties()
742 static int get_queue_data_sizes(struct kfd_process_device *pdd,  in get_queue_data_sizes()  argument
749 	ret = pqm_get_queue_checkpoint_info(&pdd->process->pqm,  in get_queue_data_sizes()
772 		struct kfd_process_device *pdd = p->pdds[i];  in kfd_process_get_queue_info()  local
774 		list_for_each_entry(q, &pdd->qpd.queues_list, list) {  in kfd_process_get_queue_info()
782 				ret = get_queue_data_sizes(pdd, q, &mqd_size, &ctl_stack_size);  in kfd_process_get_queue_info()
821 static int criu_checkpoint_queue(struct kfd_process_device *pdd,  in criu_checkpoint_queue()  argument
831 	q_data->gpu_id = pdd->user_gpu_id;  in criu_checkpoint_queue()
858 	ret = pqm_checkpoint_mqd(&pdd->process->pqm, q->properties.queue_id, mqd, ctl_stack);  in criu_checkpoint_queue()
868 static int criu_checkpoint_queues_device(struct kfd_process_device *pdd,  in criu_checkpoint_queues_device()  argument
878 	list_for_each_entry(q, &pdd->qpd.queues_list, list) {  in criu_checkpoint_queues_device()
893 		ret = get_queue_data_sizes(pdd, q, &mqd_size, &ctl_stack_size);  in criu_checkpoint_queues_device()
917 		ret = criu_checkpoint_queue(pdd, q, q_data);  in criu_checkpoint_queues_device()
945 		struct kfd_process_device *pdd = p->pdds[pdd_index];  in kfd_criu_checkpoint_queues()  local
951 		ret = criu_checkpoint_queues_device(pdd, user_priv_data, &q_index,  in kfd_criu_checkpoint_queues()
987 	struct kfd_process_device *pdd;  in kfd_criu_restore_queue()  local
1028 	pdd = kfd_process_device_data_by_id(p, q_data->gpu_id);  in kfd_criu_restore_queue()
1029 	if (!pdd) {  in kfd_criu_restore_queue()
1044 	ret = pqm_create_queue(&p->pqm, pdd->dev, &qp, &queue_id, q_data, mqd, ctl_stack, NULL);  in kfd_criu_restore_queue()
1051 		ret = pqm_set_gws(&p->pqm, q_data->q_id, pdd->dev->gws);  in kfd_criu_restore_queue()