76 static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd);
89 	struct kfd_process_device *pdd;  member
103 	struct kfd_process_device *pdd;  in kfd_sdma_activity_worker()  local
116 	pdd = workarea->pdd;  in kfd_sdma_activity_worker()
117 	if (!pdd)  in kfd_sdma_activity_worker()
119 	dqm = pdd->dev->dqm;  in kfd_sdma_activity_worker()
120 	qpd = &pdd->qpd;  in kfd_sdma_activity_worker()
175 		workarea->sdma_activity_counter = pdd->sdma_past_activity_counter;  in kfd_sdma_activity_worker()
185 	mm = get_task_mm(pdd->process->lead_thread);  in kfd_sdma_activity_worker()
212 	workarea->sdma_activity_counter += pdd->sdma_past_activity_counter;  in kfd_sdma_activity_worker()
273 	struct kfd_process_device *pdd = NULL;  in kfd_get_cu_occupancy()  local
278 	pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);  in kfd_get_cu_occupancy()
279 	dev = pdd->dev;  in kfd_get_cu_occupancy()
284 	proc = pdd->process;  in kfd_get_cu_occupancy()
285 	if (pdd->qpd.queue_count == 0) {  in kfd_get_cu_occupancy()
311 		    kfd_dqm_is_queue_in_process(dev->dqm, &pdd->qpd,  in kfd_get_cu_occupancy()
334 		struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,  in kfd_procfs_show()  local
336 		return snprintf(buffer, PAGE_SIZE, "%llu\n", atomic64_read(&pdd->vram_usage));  in kfd_procfs_show()
338 		struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,  in kfd_procfs_show()  local
345 		sdma_activity_work_handler.pdd = pdd;  in kfd_procfs_show()
426 		struct kfd_process_device *pdd = container_of(attr,  in kfd_procfs_stats_show()  local
431 		evict_jiffies = atomic64_read(&pdd->evict_duration_counter);  in kfd_procfs_stats_show()
451 	struct kfd_process_device *pdd;  in kfd_sysfs_counters_show()  local
454 		pdd = container_of(attr, struct kfd_process_device,  in kfd_sysfs_counters_show()
456 		return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->faults));  in kfd_sysfs_counters_show()
459 		pdd = container_of(attr, struct kfd_process_device,  in kfd_sysfs_counters_show()
461 		return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_in));  in kfd_sysfs_counters_show()
464 		pdd = container_of(attr, struct kfd_process_device,  in kfd_sysfs_counters_show()
466 		return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_out));  in kfd_sysfs_counters_show()
578 		struct kfd_process_device *pdd = p->pdds[i];  in kfd_procfs_add_sysfs_stats()  local
581 				"stats_%u", pdd->dev->id);  in kfd_procfs_add_sysfs_stats()
582 		pdd->kobj_stats = kfd_alloc_struct(pdd->kobj_stats);  in kfd_procfs_add_sysfs_stats()
583 		if (!pdd->kobj_stats)  in kfd_procfs_add_sysfs_stats()
586 		ret = kobject_init_and_add(pdd->kobj_stats,  in kfd_procfs_add_sysfs_stats()
594 			kobject_put(pdd->kobj_stats);  in kfd_procfs_add_sysfs_stats()
595 			pdd->kobj_stats = NULL;  in kfd_procfs_add_sysfs_stats()
599 		kfd_sysfs_create_file(pdd->kobj_stats, &pdd->attr_evict,  in kfd_procfs_add_sysfs_stats()
602 		if (pdd->dev->kfd2kgd->get_cu_occupancy)  in kfd_procfs_add_sysfs_stats()
603 			kfd_sysfs_create_file(pdd->kobj_stats,  in kfd_procfs_add_sysfs_stats()
604 					      &pdd->attr_cu_occupancy,  in kfd_procfs_add_sysfs_stats()
626 		struct kfd_process_device *pdd = p->pdds[i];  in kfd_procfs_add_sysfs_counters()  local
630 			"counters_%u", pdd->dev->id);  in kfd_procfs_add_sysfs_counters()
644 		pdd->kobj_counters = kobj_counters;  in kfd_procfs_add_sysfs_counters()
645 		kfd_sysfs_create_file(kobj_counters, &pdd->attr_faults,  in kfd_procfs_add_sysfs_counters()
647 		kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_in,  in kfd_procfs_add_sysfs_counters()
649 		kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_out,  in kfd_procfs_add_sysfs_counters()
667 		struct kfd_process_device *pdd = p->pdds[i];  in kfd_procfs_add_sysfs_files()  local
669 		snprintf(pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, "vram_%u",  in kfd_procfs_add_sysfs_files()
670 			 pdd->dev->id);  in kfd_procfs_add_sysfs_files()
671 		kfd_sysfs_create_file(p->kobj, &pdd->attr_vram,  in kfd_procfs_add_sysfs_files()
672 				      pdd->vram_filename);  in kfd_procfs_add_sysfs_files()
674 		snprintf(pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, "sdma_%u",  in kfd_procfs_add_sysfs_files()
675 			 pdd->dev->id);  in kfd_procfs_add_sysfs_files()
676 		kfd_sysfs_create_file(p->kobj, &pdd->attr_sdma,  in kfd_procfs_add_sysfs_files()
677 					    pdd->sdma_filename);  in kfd_procfs_add_sysfs_files()
719 			struct kfd_process_device *pdd, void **kptr)  in kfd_process_free_gpuvm()  argument
721 	struct kfd_node *dev = pdd->dev;  in kfd_process_free_gpuvm()
728 	amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->adev, mem, pdd->drm_priv);  in kfd_process_free_gpuvm()
729 	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, mem, pdd->drm_priv,  in kfd_process_free_gpuvm()
739 static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd,  in kfd_process_alloc_gpuvm()  argument
743 	struct kfd_node *kdev = pdd->dev;  in kfd_process_alloc_gpuvm()
747 						 pdd->drm_priv, mem, NULL,  in kfd_process_alloc_gpuvm()
753 			pdd->drm_priv);  in kfd_process_alloc_gpuvm()
775 	amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(kdev->adev, *mem, pdd->drm_priv);  in kfd_process_alloc_gpuvm()
778 	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->adev, *mem, pdd->drm_priv,  in kfd_process_alloc_gpuvm()
792 static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd)  in kfd_process_device_reserve_ib_mem()  argument
794 	struct qcm_process_device *qpd = &pdd->qpd;  in kfd_process_device_reserve_ib_mem()
807 	ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags,  in kfd_process_device_reserve_ib_mem()
818 static void kfd_process_device_destroy_ib_mem(struct kfd_process_device *pdd)  in kfd_process_device_destroy_ib_mem()  argument
820 	struct qcm_process_device *qpd = &pdd->qpd;  in kfd_process_device_destroy_ib_mem()
825 	kfd_process_free_gpuvm(qpd->ib_mem, pdd, &qpd->ib_kaddr);  in kfd_process_device_destroy_ib_mem()
983 static void kfd_process_device_free_bos(struct kfd_process_device *pdd)  in kfd_process_device_free_bos()  argument
985 	struct kfd_process *p = pdd->process;  in kfd_process_device_free_bos()
994 	idr_for_each_entry(&pdd->alloc_idr, mem, id) {  in kfd_process_device_free_bos()
1005 		amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev, mem,  in kfd_process_device_free_bos()
1006 						       pdd->drm_priv, NULL);  in kfd_process_device_free_bos()
1007 		kfd_process_device_remove_obj_handle(pdd, id);  in kfd_process_device_free_bos()
1017 	struct kfd_process_device *pdd;  in kfd_process_kunmap_signal_bo()  local
1027 	pdd = kfd_get_process_device_data(kdev, p);  in kfd_process_kunmap_signal_bo()
1028 	if (!pdd)  in kfd_process_kunmap_signal_bo()
1032 		pdd, GET_IDR_HANDLE(p->signal_handle));  in kfd_process_kunmap_signal_bo()
1055 		struct kfd_process_device *pdd = p->pdds[i];  in kfd_process_destroy_pdds()  local
1058 			pdd->dev->id, p->lead_thread->pid);  in kfd_process_destroy_pdds()
1059 		kfd_process_device_destroy_cwsr_dgpu(pdd);  in kfd_process_destroy_pdds()
1060 		kfd_process_device_destroy_ib_mem(pdd);  in kfd_process_destroy_pdds()
1062 		if (pdd->drm_file)  in kfd_process_destroy_pdds()
1063 			fput(pdd->drm_file);  in kfd_process_destroy_pdds()
1065 		if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base)  in kfd_process_destroy_pdds()
1066 			free_pages((unsigned long)pdd->qpd.cwsr_kaddr,  in kfd_process_destroy_pdds()
1069 		idr_destroy(&pdd->alloc_idr);  in kfd_process_destroy_pdds()
1071 		kfd_free_process_doorbells(pdd->dev->kfd, pdd);  in kfd_process_destroy_pdds()
1073 		if (pdd->dev->kfd->shared_resources.enable_mes &&  in kfd_process_destroy_pdds()
1074 			pdd->proc_ctx_cpu_ptr)  in kfd_process_destroy_pdds()
1075 			amdgpu_amdkfd_free_gtt_mem(pdd->dev->adev,  in kfd_process_destroy_pdds()
1076 						   &pdd->proc_ctx_bo);  in kfd_process_destroy_pdds()
1081 		if (pdd->runtime_inuse) {  in kfd_process_destroy_pdds()
1082 			pm_runtime_mark_last_busy(adev_to_drm(pdd->dev->adev)->dev);  in kfd_process_destroy_pdds()
1083 			pm_runtime_put_autosuspend(adev_to_drm(pdd->dev->adev)->dev);  in kfd_process_destroy_pdds()
1084 			pdd->runtime_inuse = false;  in kfd_process_destroy_pdds()
1087 		kfree(pdd);  in kfd_process_destroy_pdds()
1095 	struct kfd_process_device *pdd;  in kfd_process_remove_sysfs()  local
1107 		pdd = p->pdds[i];  in kfd_process_remove_sysfs()
1109 		sysfs_remove_file(p->kobj, &pdd->attr_vram);  in kfd_process_remove_sysfs()
1110 		sysfs_remove_file(p->kobj, &pdd->attr_sdma);  in kfd_process_remove_sysfs()
1112 		sysfs_remove_file(pdd->kobj_stats, &pdd->attr_evict);  in kfd_process_remove_sysfs()
1113 		if (pdd->dev->kfd2kgd->get_cu_occupancy)  in kfd_process_remove_sysfs()
1114 			sysfs_remove_file(pdd->kobj_stats,  in kfd_process_remove_sysfs()
1115 					  &pdd->attr_cu_occupancy);  in kfd_process_remove_sysfs()
1116 		kobject_del(pdd->kobj_stats);  in kfd_process_remove_sysfs()
1117 		kobject_put(pdd->kobj_stats);  in kfd_process_remove_sysfs()
1118 		pdd->kobj_stats = NULL;  in kfd_process_remove_sysfs()
1122 		pdd = p->pdds[i];  in kfd_process_remove_sysfs()
1124 		sysfs_remove_file(pdd->kobj_counters, &pdd->attr_faults);  in kfd_process_remove_sysfs()
1125 		sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_in);  in kfd_process_remove_sysfs()
1126 		sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_out);  in kfd_process_remove_sysfs()
1127 		kobject_del(pdd->kobj_counters);  in kfd_process_remove_sysfs()
1128 		kobject_put(pdd->kobj_counters);  in kfd_process_remove_sysfs()
1129 		pdd->kobj_counters = NULL;  in kfd_process_remove_sysfs()
1223 		struct kfd_process_device *pdd = p->pdds[i];  in kfd_process_notifier_release_internal()  local
1226 		if (!kfd_dbg_is_rlc_restore_supported(pdd->dev) && p->runtime_info.ttmp_setup)  in kfd_process_notifier_release_internal()
1227 			amdgpu_gfx_off_ctrl(pdd->dev->adev, true);  in kfd_process_notifier_release_internal()
1374 static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)  in kfd_process_device_init_cwsr_dgpu()  argument
1376 	struct kfd_node *dev = pdd->dev;  in kfd_process_device_init_cwsr_dgpu()
1377 	struct qcm_process_device *qpd = &pdd->qpd;  in kfd_process_device_init_cwsr_dgpu()
1389 	ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base,  in kfd_process_device_init_cwsr_dgpu()
1400 	kfd_process_set_trap_debug_flag(&pdd->qpd,  in kfd_process_device_init_cwsr_dgpu()
1401 					pdd->process->debug_trap_enabled);  in kfd_process_device_init_cwsr_dgpu()
1410 static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd)  in kfd_process_device_destroy_cwsr_dgpu()  argument
1412 	struct kfd_node *dev = pdd->dev;  in kfd_process_device_destroy_cwsr_dgpu()
1413 	struct qcm_process_device *qpd = &pdd->qpd;  in kfd_process_device_destroy_cwsr_dgpu()
1418 	kfd_process_free_gpuvm(qpd->cwsr_mem, pdd, &qpd->cwsr_kaddr);  in kfd_process_device_destroy_cwsr_dgpu()
1613 	struct kfd_process_device *pdd = NULL;  in kfd_create_process_device_data()  local
1617 	pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);  in kfd_create_process_device_data()
1618 	if (!pdd)  in kfd_create_process_device_data()
1621 	pdd->dev = dev;  in kfd_create_process_device_data()
1622 	INIT_LIST_HEAD(&pdd->qpd.queues_list);  in kfd_create_process_device_data()
1623 	INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);  in kfd_create_process_device_data()
1624 	pdd->qpd.dqm = dev->dqm;  in kfd_create_process_device_data()
1625 	pdd->qpd.pqm = &p->pqm;  in kfd_create_process_device_data()
1626 	pdd->qpd.evicted = 0;  in kfd_create_process_device_data()
1627 	pdd->qpd.mapped_gws_queue = false;  in kfd_create_process_device_data()
1628 	pdd->process = p;  in kfd_create_process_device_data()
1629 	pdd->bound = PDD_UNBOUND;  in kfd_create_process_device_data()
1630 	pdd->already_dequeued = false;  in kfd_create_process_device_data()
1631 	pdd->runtime_inuse = false;  in kfd_create_process_device_data()
1632 	atomic64_set(&pdd->vram_usage, 0);  in kfd_create_process_device_data()
1633 	pdd->sdma_past_activity_counter = 0;  in kfd_create_process_device_data()
1634 	pdd->user_gpu_id = dev->id;  in kfd_create_process_device_data()
1635 	atomic64_set(&pdd->evict_duration_counter, 0);  in kfd_create_process_device_data()
1637 	p->pdds[p->n_pdds++] = pdd;  in kfd_create_process_device_data()
1638 	if (kfd_dbg_is_per_vmid_supported(pdd->dev))  in kfd_create_process_device_data()
1639 		pdd->spi_dbg_override = pdd->dev->kfd2kgd->disable_debug_trap(  in kfd_create_process_device_data()
1640 							pdd->dev->adev,  in kfd_create_process_device_data()
1645 	idr_init(&pdd->alloc_idr);  in kfd_create_process_device_data()
1647 	return pdd;  in kfd_create_process_device_data()
1664 int kfd_process_device_init_vm(struct kfd_process_device *pdd,  in kfd_process_device_init_vm()  argument
1677 	if (pdd->drm_priv)  in kfd_process_device_init_vm()
1685 	p = pdd->process;  in kfd_process_device_init_vm()
1686 	dev = pdd->dev;  in kfd_process_device_init_vm()
1699 	pdd->drm_priv = drm_file->private_data;  in kfd_process_device_init_vm()
1701 	ret = kfd_process_device_reserve_ib_mem(pdd);  in kfd_process_device_init_vm()
1704 	ret = kfd_process_device_init_cwsr_dgpu(pdd);  in kfd_process_device_init_vm()
1709 		dev_warn(pdd->dev->adev->dev, "WARN: vm %p has no pasid associated",  in kfd_process_device_init_vm()
1715 	pdd->pasid = avm->pasid;  in kfd_process_device_init_vm()
1716 	pdd->drm_file = drm_file;  in kfd_process_device_init_vm()
1721 	kfd_process_device_destroy_cwsr_dgpu(pdd);  in kfd_process_device_init_vm()
1723 	kfd_process_device_destroy_ib_mem(pdd);  in kfd_process_device_init_vm()
1725 	pdd->drm_priv = NULL;  in kfd_process_device_init_vm()
1741 	struct kfd_process_device *pdd;  in kfd_bind_process_to_device()  local
1744 	pdd = kfd_get_process_device_data(dev, p);  in kfd_bind_process_to_device()
1745 	if (!pdd) {  in kfd_bind_process_to_device()
1750 	if (!pdd->drm_priv)  in kfd_bind_process_to_device()
1758 	if (!pdd->runtime_inuse) {  in kfd_bind_process_to_device()
1770 	pdd->runtime_inuse = true;  in kfd_bind_process_to_device()
1772 	return pdd;  in kfd_bind_process_to_device()
1778 int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,  in kfd_process_device_create_obj_handle()  argument
1781 	return idr_alloc(&pdd->alloc_idr, mem, 0, 0, GFP_KERNEL);  in kfd_process_device_create_obj_handle()
1787 void *kfd_process_device_translate_handle(struct kfd_process_device *pdd,  in kfd_process_device_translate_handle()  argument
1793 	return idr_find(&pdd->alloc_idr, handle);  in kfd_process_device_translate_handle()
1799 void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,  in kfd_process_device_remove_obj_handle()  argument
1803 		idr_remove(&pdd->alloc_idr, handle);  in kfd_process_device_remove_obj_handle()
1828 						struct kfd_process_device **pdd)  in kfd_lookup_process_by_pasid()  argument
1836 		if (pdd)  in kfd_lookup_process_by_pasid()
1837 			*pdd = ret_p;  in kfd_lookup_process_by_pasid()
1846 	if (pdd)  in kfd_lookup_process_by_pasid()
1847 		*pdd = NULL;  in kfd_lookup_process_by_pasid()
1880 		struct kfd_process_device *pdd = p->pdds[i];  in kfd_process_evict_queues()  local
1881 		struct device *dev = pdd->dev->adev->dev;  in kfd_process_evict_queues()
1883 		kfd_smi_event_queue_eviction(pdd->dev, p->lead_thread->pid,  in kfd_process_evict_queues()
1886 		r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm,  in kfd_process_evict_queues()
1887 							    &pdd->qpd);  in kfd_process_evict_queues()
1898 		pdd->dev->dqm->is_hws_hang = false;  in kfd_process_evict_queues()
1908 		struct kfd_process_device *pdd = p->pdds[i];  in kfd_process_evict_queues()  local
1913 		kfd_smi_event_queue_restore(pdd->dev, p->lead_thread->pid);  in kfd_process_evict_queues()
1915 		if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,  in kfd_process_evict_queues()
1916 							      &pdd->qpd))  in kfd_process_evict_queues()
1917 			dev_err(pdd->dev->adev->dev,  in kfd_process_evict_queues()
1933 		struct kfd_process_device *pdd = p->pdds[i];  in kfd_process_restore_queues()  local
1934 		struct device *dev = pdd->dev->adev->dev;  in kfd_process_restore_queues()
1936 		kfd_smi_event_queue_restore(pdd->dev, p->lead_thread->pid);  in kfd_process_restore_queues()
1938 		r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,  in kfd_process_restore_queues()
1939 							      &pdd->qpd);  in kfd_process_restore_queues()
2116 	struct kfd_process_device *pdd;  in kfd_reserved_mem_mmap()  local
2124 	pdd = kfd_get_process_device_data(dev, process);  in kfd_reserved_mem_mmap()
2125 	if (!pdd)  in kfd_reserved_mem_mmap()
2127 	qpd = &pdd->qpd;  in kfd_reserved_mem_mmap()
2146 int kfd_process_drain_interrupts(struct kfd_process_device *pdd)  in kfd_process_drain_interrupts()  argument
2152 	if (!KFD_IS_SOC15(pdd->dev))  in kfd_process_drain_interrupts()
2155 	pdd->process->irq_drain_is_open = true;  in kfd_process_drain_interrupts()
2160 	irq_drain_fence[3] = pdd->pasid;  in kfd_process_drain_interrupts()
2165 	if (KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 4, 3) ||  in kfd_process_drain_interrupts()
2166 	    KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 4, 4) ||  in kfd_process_drain_interrupts()
2167 	    KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 5, 0)) {  in kfd_process_drain_interrupts()
2168 		node_id = ffs(pdd->dev->interrupt_bitmap) - 1;  in kfd_process_drain_interrupts()
2173 	if (amdgpu_amdkfd_send_close_event_drain_irq(pdd->dev->adev,  in kfd_process_drain_interrupts()
2175 		pdd->process->irq_drain_is_open = false;  in kfd_process_drain_interrupts()
2179 	r = wait_event_interruptible(pdd->process->wait_irq_drain,  in kfd_process_drain_interrupts()
2180 				     !READ_ONCE(pdd->process->irq_drain_is_open));  in kfd_process_drain_interrupts()
2182 		pdd->process->irq_drain_is_open = false;  in kfd_process_drain_interrupts()
2276 			struct kfd_process_device *pdd = p->pdds[i];  in kfd_process_device_data_by_id()  local
2278 			if (pdd->user_gpu_id == gpu_id)  in kfd_process_device_data_by_id()
2279 				return pdd;  in kfd_process_device_data_by_id()
2293 		struct kfd_process_device *pdd = p->pdds[i];  in kfd_process_get_user_gpu_id()  local
2295 		if (pdd->dev->id == actual_gpu_id)  in kfd_process_get_user_gpu_id()
2296 			return pdd->user_gpu_id;  in kfd_process_get_user_gpu_id()